From 1595317e0611b0bf3ac8300ab128a43597e42c5a Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Tue, 8 Sep 2026 19:32:32 -0700 Subject: [PATCH 01/13] sandbox: honor runtime_path for the shim sandboxer A runtime handler can name its shim binary with runtime_path. The task service replaces the runtime name with that path and the podsandbox controller passes it to the pause task, but the shim sandboxer never saw it: CRI stored only the runtime type in the sandbox record and the shim controller started the binary resolved from that type. Add a RuntimePath create option to the sandbox controller, have the shim controller start that binary when it is set (the shim manager already accepts an absolute path in place of a runtime name), and pass it from CRI. The gRPC proxy cannot carry it, since ControllerCreateRequest has no field for it, so a remote sandboxer logs a warning instead. Signed-off-by: Maksym Pavlenko --- core/sandbox/controller.go | 20 +++++++++++++ core/sandbox/controller_test.go | 42 +++++++++++++++++++++++++++ core/sandbox/proxy/controller.go | 13 ++++++++- core/sandbox/proxy/controller_test.go | 35 ++++++++++++++++++++++ internal/cri/server/sandbox_run.go | 2 +- plugins/sandbox/controller.go | 12 ++++++-- plugins/sandbox/controller_test.go | 35 ++++++++++++++++++++++ 7 files changed, 155 insertions(+), 4 deletions(-) create mode 100644 core/sandbox/controller_test.go create mode 100644 core/sandbox/proxy/controller_test.go create mode 100644 plugins/sandbox/controller_test.go diff --git a/core/sandbox/controller.go b/core/sandbox/controller.go index 2849883daa..7a4b41d203 100644 --- a/core/sandbox/controller.go +++ b/core/sandbox/controller.go @@ -19,10 +19,12 @@ package sandbox import ( "context" "fmt" + "path/filepath" "time" "github.com/containerd/containerd/api/types" "github.com/containerd/containerd/v2/core/mount" + "github.com/containerd/errdefs" "github.com/containerd/typeurl/v2" imagespec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -36,10 +38,28 @@ type CreateOptions struct { Options typeurl.Any NetNSPath string Annotations map[string]string + // RuntimePath is an absolute path to the shim binary to start for the + // sandbox instead of resolving it from the runtime name, the same way a + // task can be created with a runtime path. + RuntimePath string } type CreateOpt func(*CreateOptions) error +// WithRuntimePath starts the sandbox with the shim binary at the given +// absolute path instead of the one resolved from the runtime name. An empty +// path keeps the runtime name; a relative one is rejected, since the shim +// manager would take it for a runtime name. +func WithRuntimePath(path string) CreateOpt { + return func(co *CreateOptions) error { + if path != "" && !filepath.IsAbs(path) { + return fmt.Errorf("runtime path %q is not absolute: %w", path, errdefs.ErrInvalidArgument) + } + co.RuntimePath = path + return nil + } +} + // WithRootFS is used to create a sandbox with the provided rootfs mount func WithRootFS(m []mount.Mount) CreateOpt { return func(co *CreateOptions) error { diff --git a/core/sandbox/controller_test.go b/core/sandbox/controller_test.go new file mode 100644 index 0000000000..d5137f41e3 --- /dev/null +++ b/core/sandbox/controller_test.go @@ -0,0 +1,42 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "path/filepath" + "testing" + + "github.com/containerd/errdefs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWithRuntimePath(t *testing.T) { + var opts CreateOptions + require.NoError(t, WithRuntimePath("")(&opts)) + assert.Empty(t, opts.RuntimePath, "an empty path keeps the runtime name") + + // An absolute path of this platform (a leading slash is not absolute on + // Windows). + abs := filepath.Join(t.TempDir(), "containerd-shim-example-v1") + require.NoError(t, WithRuntimePath(abs)(&opts)) + assert.Equal(t, abs, opts.RuntimePath) + + err := WithRuntimePath("containerd-shim-example-v1")(&opts) + require.Error(t, err, "a relative path would be taken for a runtime name") + assert.True(t, errdefs.IsInvalidArgument(err)) +} diff --git a/core/sandbox/proxy/controller.go b/core/sandbox/proxy/controller.go index 5f8ca44843..2c8ae55ed3 100644 --- a/core/sandbox/proxy/controller.go +++ b/core/sandbox/proxy/controller.go @@ -24,6 +24,7 @@ import ( "github.com/containerd/containerd/api/types" "github.com/containerd/errdefs" "github.com/containerd/errdefs/pkg/errgrpc" + "github.com/containerd/log" "github.com/containerd/typeurl/v2" imagespec "github.com/opencontainers/image-spec/specs-go/v1" @@ -47,7 +48,17 @@ func NewSandboxController(client api.ControllerClient, name string) sandbox.Cont func (s *remoteSandboxController) Create(ctx context.Context, sandboxInfo sandbox.Sandbox, opts ...sandbox.CreateOpt) error { var options sandbox.CreateOptions for _, opt := range opts { - opt(&options) + if err := opt(&options); err != nil { + return err + } + } + if options.RuntimePath != "" { + // ControllerCreateRequest has no field for it; the remote controller + // starts whatever the runtime name resolves to. + log.G(ctx).WithFields(log.Fields{ + "runtime_path": options.RuntimePath, + "runtime": sandboxInfo.Runtime.Name, + }).Warn("runtime path is not forwarded to a remote sandbox controller, which resolves the runtime name instead") } apiSandbox := sandbox.ToProto(&sandboxInfo) _, err := s.client.Create(ctx, &api.ControllerCreateRequest{ diff --git a/core/sandbox/proxy/controller_test.go b/core/sandbox/proxy/controller_test.go new file mode 100644 index 0000000000..78dbb30e4e --- /dev/null +++ b/core/sandbox/proxy/controller_test.go @@ -0,0 +1,35 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package proxy + +import ( + "context" + "testing" + + "github.com/containerd/errdefs" + "github.com/stretchr/testify/require" + + "github.com/containerd/containerd/v2/core/sandbox" +) + +// A failing create option ends Create before the request is sent: the +// controller has no client here, so getting past the options would panic. +func TestCreateOptionError(t *testing.T) { + c := NewSandboxController(nil, "remote") + err := c.Create(context.Background(), sandbox.Sandbox{ID: "sb"}, sandbox.WithRuntimePath("containerd-shim-example-v1")) + require.ErrorIs(t, err, errdefs.ErrInvalidArgument) +} diff --git a/internal/cri/server/sandbox_run.go b/internal/cri/server/sandbox_run.go index 1972888a7f..14bd9ec256 100644 --- a/internal/cri/server/sandbox_run.go +++ b/internal/cri/server/sandbox_run.go @@ -304,7 +304,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox } } - if err := c.sandboxService.CreateSandbox(ctx, sandboxInfo, sb.WithOptions(config), sb.WithNetNSPath(sandbox.NetNSPath)); err != nil { + if err := c.sandboxService.CreateSandbox(ctx, sandboxInfo, sb.WithOptions(config), sb.WithNetNSPath(sandbox.NetNSPath), sb.WithRuntimePath(ociRuntime.Path)); err != nil { return nil, fmt.Errorf("failed to create sandbox %q: %w", id, err) } diff --git a/plugins/sandbox/controller.go b/plugins/sandbox/controller.go index cee78ad96b..823af3ec9b 100644 --- a/plugins/sandbox/controller.go +++ b/plugins/sandbox/controller.go @@ -124,7 +124,9 @@ func (c *controllerLocal) Create(ctx context.Context, info sandbox.Sandbox, opts var coptions sandbox.CreateOptions sandboxID := info.ID for _, opt := range opts { - opt(&coptions) + if err := opt(&coptions); err != nil { + return err + } } if _, err := c.shims.Get(ctx, sandboxID); err == nil { @@ -141,10 +143,16 @@ func (c *controllerLocal) Create(ctx context.Context, info sandbox.Sandbox, opts } }() + // The shim manager accepts an absolute binary path in place of a + // runtime name. + runtimeName := info.Runtime.Name + if coptions.RuntimePath != "" { + runtimeName = coptions.RuntimePath + } shim, err := c.shims.Start(ctx, sandboxID, bundle, runtime.CreateOpts{ Spec: info.Spec, RuntimeOptions: info.Runtime.Options, - Runtime: info.Runtime.Name, + Runtime: runtimeName, TaskOptions: nil, }) if err != nil { diff --git a/plugins/sandbox/controller_test.go b/plugins/sandbox/controller_test.go new file mode 100644 index 0000000000..d702703fc9 --- /dev/null +++ b/plugins/sandbox/controller_test.go @@ -0,0 +1,35 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "testing" + + "github.com/containerd/errdefs" + "github.com/stretchr/testify/require" + + "github.com/containerd/containerd/v2/core/sandbox" +) + +// A failing create option ends Create before any shim is involved: the +// controller has no shim manager here, so getting past the options would panic. +func TestCreateOptionError(t *testing.T) { + c := &controllerLocal{} + err := c.Create(context.Background(), sandbox.Sandbox{ID: "sb"}, sandbox.WithRuntimePath("containerd-shim-example-v1")) + require.ErrorIs(t, err, errdefs.ErrInvalidArgument) +} From 5aaba1c85df532b51ae852b2cb2bd0e1815fa62b Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Tue, 8 Sep 2026 19:32:32 -0700 Subject: [PATCH 02/13] sandbox: convert shim metrics errors to native errors controllerLocal.Metrics returned the shim's gRPC status error as is, unlike every other method of the controller. CRI's pod stats listing tolerates NotFound and Unavailable from a controller but never matched a status error, so one sandbox whose shim could not report metrics failed ListPodSandboxStats for every pod on the node. Signed-off-by: Maksym Pavlenko --- plugins/sandbox/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/sandbox/controller.go b/plugins/sandbox/controller.go index 823af3ec9b..52fa8caf54 100644 --- a/plugins/sandbox/controller.go +++ b/plugins/sandbox/controller.go @@ -339,7 +339,7 @@ func (c *controllerLocal) Metrics(ctx context.Context, sandboxID string) (*types req := &runtimeAPI.SandboxMetricsRequest{SandboxID: sandboxID} resp, err := sb.SandboxMetrics(ctx, req) if err != nil { - return nil, err + return nil, errgrpc.ToNative(err) } return resp.Metrics, nil } From 3c0e78514999715f104e6b824dde05b03422583c Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Tue, 8 Sep 2026 19:32:32 -0700 Subject: [PATCH 03/13] cri: add the sandboxfiles package The files a pod shares with its containers (hostname, hosts, resolv.conf, the shm tmpfs) are produced by the podsandbox controller from the pod config. A sandbox implementation outside the CRI server, such as a Sandbox API shim, has to produce the same files. Put the pure helpers (file and mount path constants, hostname and resolv.conf content, the shm mount data) in a leaf package with no CRI server dependencies so that both can use them. podsandbox is left unchanged for now. Signed-off-by: Maksym Pavlenko --- internal/cri/sandboxfiles/sandboxfiles.go | 88 +++++++++++++++++++ .../cri/sandboxfiles/sandboxfiles_test.go | 64 ++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 internal/cri/sandboxfiles/sandboxfiles.go create mode 100644 internal/cri/sandboxfiles/sandboxfiles_test.go diff --git a/internal/cri/sandboxfiles/sandboxfiles.go b/internal/cri/sandboxfiles/sandboxfiles.go new file mode 100644 index 0000000000..3b8f38cb3a --- /dev/null +++ b/internal/cri/sandboxfiles/sandboxfiles.go @@ -0,0 +1,88 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package sandboxfiles builds the content of the files a pod sandbox shares +// with every container of the pod: /etc/hostname, /etc/hosts, /etc/resolv.conf +// and the /dev/shm tmpfs. +// +// The package is deliberately a leaf: it has no dependency on the CRI server +// or on any sandbox implementation, so that both the in-process podsandbox +// controller and a shim that implements the Sandbox API can produce the same +// files. Where the files live and how they reach the containers is decided by +// the caller. +package sandboxfiles + +import ( + "fmt" + "strings" +) + +const ( + // HostnameFile is the file name of the pod hostname file. + HostnameFile = "hostname" + // HostsFile is the file name of the pod hosts file. + HostsFile = "hosts" + // ResolvConfFile is the file name of the pod resolv.conf. + ResolvConfFile = "resolv.conf" + // ShmDir is the directory name of the pod /dev/shm tmpfs. + ShmDir = "shm" + + // EtcHostname is the path of the hostname file inside a container. + EtcHostname = "/etc/hostname" + // EtcHosts is the path of the hosts file on the host and inside a container. + EtcHosts = "/etc/hosts" + // ResolvConfPath is the path of resolv.conf on the host and inside a container. + ResolvConfPath = "/etc/resolv.conf" + // DevShm is the path of the shared memory tmpfs on the host and inside a container. + DevShm = "/dev/shm" + + // DefaultShmSize is the default size of the pod /dev/shm tmpfs. + DefaultShmSize = int64(1024 * 1024 * 64) +) + +// HostnameContent returns the content of the pod hostname file. +func HostnameContent(hostname string) []byte { + return []byte(hostname + "\n") +} + +// ResolvConfContent renders resolv.conf content from CRI DNS options. When no +// option is specified the result is empty. +func ResolvConfContent(servers, searches, options []string) []byte { + var b strings.Builder + + if len(searches) > 0 { + fmt.Fprintf(&b, "search %s\n", strings.Join(searches, " ")) + } + + if len(servers) > 0 { + fmt.Fprintf(&b, "nameserver %s\n", strings.Join(servers, "\nnameserver ")) + } + + if len(options) > 0 { + fmt.Fprintf(&b, "options %s\n", strings.Join(options, " ")) + } + + return []byte(b.String()) +} + +// ShmMountData returns the tmpfs mount data for a pod /dev/shm of the given +// size. A non-positive size falls back to DefaultShmSize. +func ShmMountData(size int64) string { + if size <= 0 { + size = DefaultShmSize + } + return fmt.Sprintf("mode=1777,size=%d", size) +} diff --git a/internal/cri/sandboxfiles/sandboxfiles_test.go b/internal/cri/sandboxfiles/sandboxfiles_test.go new file mode 100644 index 0000000000..e083d41e12 --- /dev/null +++ b/internal/cri/sandboxfiles/sandboxfiles_test.go @@ -0,0 +1,64 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandboxfiles + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestResolvConfContent(t *testing.T) { + for _, test := range []struct { + desc string + servers []string + searches []string + options []string + expected string + }{ + { + desc: "empty dns options should return empty content", + expected: "", + }, + { + desc: "non-empty dns options should return correct content", + servers: []string{"8.8.8.8", "8.8.4.4"}, + searches: []string{"114.114.114.114"}, + options: []string{"timeout:1"}, + expected: "search 114.114.114.114\nnameserver 8.8.8.8\nnameserver 8.8.4.4\noptions timeout:1\n", + }, + { + desc: "servers only", + servers: []string{"1.1.1.1"}, + expected: "nameserver 1.1.1.1\n", + }, + } { + t.Run(test.desc, func(t *testing.T) { + assert.Equal(t, test.expected, string(ResolvConfContent(test.servers, test.searches, test.options))) + }) + } +} + +func TestHostnameContent(t *testing.T) { + assert.Equal(t, "pod-1\n", string(HostnameContent("pod-1"))) +} + +func TestShmMountData(t *testing.T) { + assert.Equal(t, "mode=1777,size=67108864", ShmMountData(0)) + assert.Equal(t, "mode=1777,size=67108864", ShmMountData(-1)) + assert.Equal(t, "mode=1777,size=1024", ShmMountData(1024)) +} From ecff1401cb1032053425e9be777ab597bd083e8f Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Tue, 8 Sep 2026 19:32:32 -0700 Subject: [PATCH 04/13] cri/opts: add WithPodNamespacePaths WithPodNamespaces derives the namespaces a container joins from the pid of the sandbox process. A sandbox without a process has to name the namespaces it holds instead. WithPodNamespacePaths takes the paths explicitly. The network, IPC and UTS namespaces of a container are always those of its pod, whatever the container config says, exactly as they were with pause; the pod's namespace options decide which of them are host namespaces. Every namespace the pod shares must have a path: a missing one is an error, never a fallback to a fresh or host namespace, unless the pod asked for the host namespace, in which case the entry is removed from the spec. A sandbox may declare that it holds no PID namespace and that each container gets its own. WithPodNamespaces delegates to it with the pid derived paths, so pause based pods produce the same specs as before. Signed-off-by: Maksym Pavlenko --- internal/cri/opts/spec_opts.go | 110 +++++++- .../cri/opts/spec_opts_namespaces_test.go | 247 ++++++++++++++++++ 2 files changed, 349 insertions(+), 8 deletions(-) create mode 100644 internal/cri/opts/spec_opts_namespaces_test.go diff --git a/internal/cri/opts/spec_opts.go b/internal/cri/opts/spec_opts.go index 0d78d54673..adb7ce7c04 100644 --- a/internal/cri/opts/spec_opts.go +++ b/internal/cri/opts/spec_opts.go @@ -344,17 +344,103 @@ func WithNamespacePath(t runtimespec.LinuxNamespaceType, nsPath string) oci.Spec } } -// WithPodNamespaces sets the pod namespaces for the container +// PodNamespacePaths are the paths of the pod namespaces a container joins. +// +// A pod sandbox is either PID-backed (a process such as pause is a member of +// every pod namespace, and the paths are derived from its pid with +// PodNamespacePathsFromPid) or path-backed (the sandbox holds the namespaces +// without a process and reports their paths, typically bind-mounted namespace +// files). An empty path means the sandbox does not hold that namespace. +type PodNamespacePaths struct { + Network string + IPC string + UTS string + // PID is the pod PID namespace, joined only when the pod shares it + // (NamespaceMode_POD or NamespaceMode_TARGET). The container keeps its own + // PID namespace otherwise. It is empty for a host PID pod on a sandbox + // without a process: the containers are then in the host PID namespace. + PID string + // PrivatePID is set by a sandbox that holds no PID namespace and declares + // that each container gets a new one, even when the pod asked to share + // (a sandbox without a process has no PID 1 to hold a shared namespace). + // It is an explicit declaration in the sandbox spec, not a fallback. + PrivatePID bool + User string +} + +// PodNamespacePathsFromPid derives the pod namespace paths from the pid of a +// process that is a member of every pod namespace, e.g. the pause container. +// targetPid is the process whose PID namespace the container joins when the +// pod shares one; it differs from sandboxPid for NamespaceMode_TARGET. +func PodNamespacePathsFromPid(sandboxPid, targetPid uint32) PodNamespacePaths { + return PodNamespacePaths{ + Network: GetNetworkNamespace(sandboxPid), + IPC: GetIPCNamespace(sandboxPid), + UTS: GetUTSNamespace(sandboxPid), + PID: GetPIDNamespace(targetPid), + User: GetUserNamespace(sandboxPid), + } +} + +// WithPodNamespaces sets the pod namespaces for the container from the pid of +// the sandbox process. See WithPodNamespacePaths. func WithPodNamespaces(config *runtime.LinuxContainerSecurityContext, sandboxPid uint32, targetPid uint32, uids, gids []runtimespec.LinuxIDMapping) oci.SpecOpts { + return WithPodNamespacePaths(config, config.GetNamespaceOptions(), PodNamespacePathsFromPid(sandboxPid, targetPid), uids, gids) +} + +// WithPodNamespacePaths makes the container join the pod namespaces at the +// given paths. The network, IPC and UTS namespaces of a container are always +// those of its pod, whatever the container config says (with a pause +// container the container joined the namespaces of pause, which was itself in +// the host namespaces the pod asked for); pod is the namespace options of the +// pod sandbox and decides which of them are host namespaces. The PID and user +// namespaces follow the container config as before; a container sharing the +// pod PID namespace (NamespaceMode_POD) is in the host one when the pod is. +// +// Every namespace the container is expected to share with the pod must have a +// path; a missing path is an error rather than a fallback to a fresh or host +// namespace, so that a broken sandbox cannot silently weaken isolation. The +// entry of a host namespace (NamespaceMode_NODE) is removed from the spec +// when the sandbox provides no path for it. +func WithPodNamespacePaths(config *runtime.LinuxContainerSecurityContext, pod *runtime.NamespaceOption, paths PodNamespacePaths, uids, gids []runtimespec.LinuxIDMapping) oci.SpecOpts { namespaces := config.GetNamespaceOptions() - opts := []oci.SpecOpts{ - oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: runtimespec.NetworkNamespace, Path: GetNetworkNamespace(sandboxPid)}), - oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: runtimespec.IPCNamespace, Path: GetIPCNamespace(sandboxPid)}), - oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: runtimespec.UTSNamespace, Path: GetUTSNamespace(sandboxPid)}), + var opts []oci.SpecOpts + join := func(t runtimespec.LinuxNamespaceType, path string, host bool) error { + switch { + case path != "": + opts = append(opts, oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: t, Path: path})) + case host: + opts = append(opts, WithoutNamespace(t)) + default: + return fmt.Errorf("pod sandbox provides no %s namespace for the container to join", t) + } + return nil } - if namespaces.GetPid() != runtime.NamespaceMode_CONTAINER { - opts = append(opts, oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: runtimespec.PIDNamespace, Path: GetPIDNamespace(targetPid)})) + + hostNetwork := pod.GetNetwork() == runtime.NamespaceMode_NODE + if err := join(runtimespec.NetworkNamespace, paths.Network, hostNetwork); err != nil { + return failSpecOpt(err) + } + if err := join(runtimespec.IPCNamespace, paths.IPC, pod.GetIpc() == runtime.NamespaceMode_NODE); err != nil { + return failSpecOpt(err) + } + // There is no CRI option for the UTS namespace: a pod shares it unless it + // uses the host network, in which case it uses the host UTS namespace too. + if err := join(runtimespec.UTSNamespace, paths.UTS, hostNetwork); err != nil { + return failSpecOpt(err) + } + // The default spec already gives the container its own PID namespace, + // which is what NamespaceMode_CONTAINER asks for and what a sandbox that + // declared it holds no PID namespace to share provides. A container that + // shares the pod PID namespace is in the host one when the pod is (pause + // was): a sandbox without a process provides no path for it. + pidMode := namespaces.GetPid() + if pidMode != runtime.NamespaceMode_CONTAINER && (pidMode != runtime.NamespaceMode_POD || !paths.PrivatePID) { + hostPID := pidMode == runtime.NamespaceMode_NODE || (pidMode == runtime.NamespaceMode_POD && pod.GetPid() == runtime.NamespaceMode_NODE) + if err := join(runtimespec.PIDNamespace, paths.PID, hostPID); err != nil { + return failSpecOpt(err) + } } if namespaces.GetUsernsOptions() != nil { @@ -362,7 +448,9 @@ func WithPodNamespaces(config *runtime.LinuxContainerSecurityContext, sandboxPid case runtime.NamespaceMode_NODE: // Nothing to do. Not adding userns field uses the node userns. case runtime.NamespaceMode_POD: - opts = append(opts, oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: runtimespec.UserNamespace, Path: GetUserNamespace(sandboxPid)})) + if err := join(runtimespec.UserNamespace, paths.User, false); err != nil { + return failSpecOpt(err) + } opts = append(opts, oci.WithUserNamespace(uids, gids)) } } @@ -370,6 +458,12 @@ func WithPodNamespaces(config *runtime.LinuxContainerSecurityContext, sandboxPid return oci.Compose(opts...) } +func failSpecOpt(err error) oci.SpecOpts { + return func(context.Context, oci.Client, *containers.Container, *runtimespec.Spec) error { + return err + } +} + const ( // netNSFormat is the format of network namespace of a process. netNSFormat = "/proc/%v/ns/net" diff --git a/internal/cri/opts/spec_opts_namespaces_test.go b/internal/cri/opts/spec_opts_namespaces_test.go new file mode 100644 index 0000000000..6d9c568269 --- /dev/null +++ b/internal/cri/opts/spec_opts_namespaces_test.go @@ -0,0 +1,247 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package opts + +import ( + "context" + "testing" + + runtimespec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/containerd/containerd/v2/pkg/oci" +) + +// defaultNamespaces are the namespaces of the default OCI spec: private, no +// path, before the pod namespaces are applied. +func defaultNamespaces() *runtimespec.Spec { + return &runtimespec.Spec{Linux: &runtimespec.Linux{Namespaces: []runtimespec.LinuxNamespace{ + {Type: runtimespec.PIDNamespace}, + {Type: runtimespec.IPCNamespace}, + {Type: runtimespec.UTSNamespace}, + {Type: runtimespec.MountNamespace}, + {Type: runtimespec.NetworkNamespace}, + }}} +} + +func applyNamespaces(t *testing.T, opt oci.SpecOpts) (*runtimespec.Spec, error) { + t.Helper() + spec := defaultNamespaces() + err := opt(context.Background(), nil, nil, spec) + return spec, err +} + +func namespacePaths(spec *runtimespec.Spec) map[runtimespec.LinuxNamespaceType]string { + paths := make(map[runtimespec.LinuxNamespaceType]string) + for _, ns := range spec.Linux.Namespaces { + paths[ns.Type] = ns.Path + } + return paths +} + +func securityContext(network, ipc, pid runtime.NamespaceMode) *runtime.LinuxContainerSecurityContext { + return &runtime.LinuxContainerSecurityContext{NamespaceOptions: &runtime.NamespaceOption{ + Network: network, + Ipc: ipc, + Pid: pid, + }} +} + +func TestWithPodNamespacePathsFromPid(t *testing.T) { + // A sandbox with a process (pause) keeps today's construction exactly, + // whatever the requested modes: the container joins the namespaces of the + // sandbox process, which is itself in the host namespaces the pod asked for. + for name, sc := range map[string]*runtime.LinuxContainerSecurityContext{ + "pod namespaces": securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_POD), + "host namespaces": securityContext(runtime.NamespaceMode_NODE, runtime.NamespaceMode_NODE, runtime.NamespaceMode_NODE), + "container pid": securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER), + } { + t.Run(name, func(t *testing.T) { + legacy, err := applyNamespaces(t, WithPodNamespaces(sc, 42, 42, nil, nil)) + require.NoError(t, err) + byPath, err := applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), PodNamespacePathsFromPid(42, 42), nil, nil)) + require.NoError(t, err) + assert.Equal(t, legacy, byPath) + + paths := namespacePaths(byPath) + assert.Equal(t, "/proc/42/ns/net", paths[runtimespec.NetworkNamespace]) + assert.Equal(t, "/proc/42/ns/ipc", paths[runtimespec.IPCNamespace]) + assert.Equal(t, "/proc/42/ns/uts", paths[runtimespec.UTSNamespace]) + if sc.GetNamespaceOptions().GetPid() == runtime.NamespaceMode_CONTAINER { + assert.Equal(t, "", paths[runtimespec.PIDNamespace], "the container keeps its own PID namespace") + } else { + assert.Equal(t, "/proc/42/ns/pid", paths[runtimespec.PIDNamespace]) + } + }) + } + + t.Run("target pid", func(t *testing.T) { + sc := securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_TARGET) + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), PodNamespacePathsFromPid(42, 7), nil, nil)) + require.NoError(t, err) + assert.Equal(t, "/proc/7/ns/pid", namespacePaths(spec)[runtimespec.PIDNamespace]) + }) +} + +func TestWithPodNamespacePaths(t *testing.T) { + pathBacked := PodNamespacePaths{ + Network: "/var/run/netns/cni-1", + IPC: "/run/sandbox/ns/ipc", + UTS: "/run/sandbox/ns/uts", + } + + t.Run("pod namespaces by path", func(t *testing.T) { + sc := securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER) + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), pathBacked, nil, nil)) + require.NoError(t, err) + assert.Equal(t, map[runtimespec.LinuxNamespaceType]string{ + runtimespec.NetworkNamespace: pathBacked.Network, + runtimespec.IPCNamespace: pathBacked.IPC, + runtimespec.UTSNamespace: pathBacked.UTS, + runtimespec.PIDNamespace: "", + runtimespec.MountNamespace: "", + }, namespacePaths(spec)) + }) + + t.Run("host namespaces without paths are removed from the spec", func(t *testing.T) { + sc := securityContext(runtime.NamespaceMode_NODE, runtime.NamespaceMode_NODE, runtime.NamespaceMode_NODE) + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), PodNamespacePaths{}, nil, nil)) + require.NoError(t, err) + assert.Equal(t, map[runtimespec.LinuxNamespaceType]string{ + runtimespec.MountNamespace: "", + }, namespacePaths(spec), "only the private mount namespace is left") + }) + + t.Run("a sandbox declaring private PID namespaces", func(t *testing.T) { + // A pod asking to share its PID namespace on a sandbox that holds + // none: the sandbox said so explicitly, the container keeps its own. + sc := securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_POD) + paths := pathBacked + paths.PrivatePID = true + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), paths, nil, nil)) + require.NoError(t, err) + assert.Equal(t, "", namespacePaths(spec)[runtimespec.PIDNamespace]) + assert.Contains(t, namespacePaths(spec), runtimespec.PIDNamespace) + + // The declaration does not cover the host PID namespace. + sc = securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_NODE) + spec, err = applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), paths, nil, nil)) + require.NoError(t, err) + assert.NotContains(t, namespacePaths(spec), runtimespec.PIDNamespace) + + // Nor a container joining the PID namespace of another container. + sc = securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_TARGET) + paths.PID = "/proc/7/ns/pid" + spec, err = applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), paths, nil, nil)) + require.NoError(t, err) + assert.Equal(t, paths.PID, namespacePaths(spec)[runtimespec.PIDNamespace]) + }) + + t.Run("host pid pod", func(t *testing.T) { + // A sandbox without a process holds no PID namespace for a host PID + // pod and provides no path: a container sharing the pod PID namespace + // (NamespaceMode_POD, the zero value) is in the host one like the + // pod, as it was with pause; NamespaceMode_CONTAINER keeps its own. + pod := &runtime.NamespaceOption{Network: runtime.NamespaceMode_POD, Ipc: runtime.NamespaceMode_POD, Pid: runtime.NamespaceMode_NODE} + for _, mode := range []runtime.NamespaceMode{runtime.NamespaceMode_POD, runtime.NamespaceMode_NODE} { + sc := securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, mode) + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, pod, pathBacked, nil, nil)) + require.NoError(t, err, "pid mode %s", mode) + assert.NotContains(t, namespacePaths(spec), runtimespec.PIDNamespace, "pid mode %s", mode) + } + sc := securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER) + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, pod, pathBacked, nil, nil)) + require.NoError(t, err) + assert.Equal(t, "", namespacePaths(spec)[runtimespec.PIDNamespace]) + assert.Contains(t, namespacePaths(spec), runtimespec.PIDNamespace) + }) + + t.Run("host network with pod ipc", func(t *testing.T) { + sc := securityContext(runtime.NamespaceMode_NODE, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER) + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), PodNamespacePaths{IPC: pathBacked.IPC}, nil, nil)) + require.NoError(t, err) + assert.Equal(t, map[runtimespec.LinuxNamespaceType]string{ + runtimespec.IPCNamespace: pathBacked.IPC, + runtimespec.PIDNamespace: "", + runtimespec.MountNamespace: "", + }, namespacePaths(spec), "no network and no UTS namespace entry") + }) + + t.Run("the pod decides the host namespaces, not the container config", func(t *testing.T) { + // A container config that says nothing (NamespaceMode_POD, the zero + // value) in a host network pod: the container is in the host network + // and UTS namespaces like the pod, as it was with pause. + container := securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER) + pod := &runtime.NamespaceOption{Network: runtime.NamespaceMode_NODE, Ipc: runtime.NamespaceMode_POD} + spec, err := applyNamespaces(t, WithPodNamespacePaths(container, pod, PodNamespacePaths{IPC: pathBacked.IPC}, nil, nil)) + require.NoError(t, err) + assert.Equal(t, map[runtimespec.LinuxNamespaceType]string{ + runtimespec.IPCNamespace: pathBacked.IPC, + runtimespec.PIDNamespace: "", + runtimespec.MountNamespace: "", + }, namespacePaths(spec)) + }) + + // A pod namespace the sandbox does not provide is an error, never a + // silent fallback to a fresh or host namespace. + for name, tc := range map[string]struct { + sc *runtime.LinuxContainerSecurityContext + paths PodNamespacePaths + }{ + "missing network": {securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER), PodNamespacePaths{IPC: pathBacked.IPC, UTS: pathBacked.UTS}}, + "missing ipc": {securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER), PodNamespacePaths{Network: pathBacked.Network, UTS: pathBacked.UTS}}, + "missing uts": {securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER), PodNamespacePaths{Network: pathBacked.Network, IPC: pathBacked.IPC}}, + "missing pid": {securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_POD), pathBacked}, + "missing target pid": {securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_TARGET), PodNamespacePaths{ + Network: pathBacked.Network, + IPC: pathBacked.IPC, + UTS: pathBacked.UTS, + PrivatePID: true, + }}, + "missing user": {&runtime.LinuxContainerSecurityContext{NamespaceOptions: &runtime.NamespaceOption{ + Network: runtime.NamespaceMode_POD, + Ipc: runtime.NamespaceMode_POD, + Pid: runtime.NamespaceMode_CONTAINER, + UsernsOptions: &runtime.UserNamespace{Mode: runtime.NamespaceMode_POD}, + }}, pathBacked}, + } { + t.Run(name, func(t *testing.T) { + _, err := applyNamespaces(t, WithPodNamespacePaths(tc.sc, tc.sc.GetNamespaceOptions(), tc.paths, nil, nil)) + require.Error(t, err) + assert.Contains(t, err.Error(), "provides no") + }) + } + + t.Run("pod user namespace by path", func(t *testing.T) { + sc := &runtime.LinuxContainerSecurityContext{NamespaceOptions: &runtime.NamespaceOption{ + Network: runtime.NamespaceMode_POD, + Ipc: runtime.NamespaceMode_POD, + Pid: runtime.NamespaceMode_CONTAINER, + UsernsOptions: &runtime.UserNamespace{Mode: runtime.NamespaceMode_POD}, + }} + paths := pathBacked + paths.User = "/run/sandbox/ns/user" + uids := []runtimespec.LinuxIDMapping{{ContainerID: 0, HostID: 65536, Size: 65536}} + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), paths, uids, uids)) + require.NoError(t, err) + assert.Equal(t, paths.User, namespacePaths(spec)[runtimespec.UserNamespace]) + assert.Equal(t, uids, spec.Linux.UIDMappings) + assert.Equal(t, uids, spec.Linux.GIDMappings) + }) +} From be6ce30fcec7497fb23c503b03120f8794a141ce Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Tue, 8 Sep 2026 19:32:32 -0700 Subject: [PATCH 05/13] cri: join a sandbox without a process through its spec A sandbox controller that reports pid 0 has no process whose /proc//ns entries a container could join and no pause container whose shared files CRI created. Such a sandbox describes what it holds in the spec it returns on start. When the sandbox status reports pid 0, load that spec from the sandbox store and use it in two places. The pod namespaces come from its Linux.Namespaces: a namespace listed there is joined by path, one that is missing is the host namespace when the pod asked for it and an error otherwise, and a PID namespace entry without a path declares that each container gets its own. The pod shared files (/etc/hostname, /etc/hosts, /etc/resolv.conf, /dev/shm) are bind-mounted from the spec mount sources when nothing exists at the fixed CRI paths, which keep precedence. Sandboxes with a pid keep the pid derived construction unchanged, and so do Sandbox API shims that report pid 0 without a Linux section in their spec, so existing shims see no change. Signed-off-by: Maksym Pavlenko --- internal/cri/server/container_create.go | 120 ++++++++++++- .../cri/server/container_create_linux_test.go | 38 ++-- ...container_create_sandboxspec_linux_test.go | 168 ++++++++++++++++++ internal/cri/server/container_create_test.go | 8 +- .../server/container_create_windows_test.go | 10 +- 5 files changed, 307 insertions(+), 37 deletions(-) create mode 100644 internal/cri/server/container_create_sandboxspec_linux_test.go diff --git a/internal/cri/server/container_create.go b/internal/cri/server/container_create.go index a3864be214..17cc9deac4 100644 --- a/internal/cri/server/container_create.go +++ b/internal/cri/server/container_create.go @@ -78,6 +78,15 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta if sandbox.Status.Get().State != sandboxstore.StateReady { return nil, fmt.Errorf("sandbox container %q is not running", sandboxID) } + // A sandbox without a process (pid 0) describes the namespaces and files it + // holds in the spec it returned on start; containers are wired up from it. + var sandboxSpec *runtimespec.Spec + if sandboxPid == 0 { + sandboxSpec, err = c.sandboxSpec(ctx, sandboxID) + if err != nil { + return nil, err + } + } span.SetAttributes( tracing.Attribute("sandbox.id", sandboxID), tracing.Attribute("sandbox.pid", sandboxPid), @@ -153,6 +162,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta podSandboxConfig: sandboxConfig, sandboxRuntimeHandler: sandbox.Metadata.RuntimeHandler, sandboxPid: sandboxPid, + sandboxSpec: sandboxSpec, NetNSPath: sandbox.NetNSPath, containerName: containerName, containerdImage: &containerdImage, @@ -183,6 +193,10 @@ type createContainerRequest struct { containerdImage *containerd.Image meta *containerstore.Metadata start time.Time + // sandboxSpec is the spec returned by the sandbox controller on start. It is + // only loaded for sandboxes without a process (sandboxPid == 0), where it is + // the source of the pod namespace paths and shared file mounts. + sandboxSpec *runtimespec.Spec } func (c *criService) createContainer(r *createContainerRequest) (_ string, retErr error) { @@ -275,6 +289,7 @@ func (c *criService) createContainer(r *createContainerRequest) (_ string, retEr volumeMounts, ociRuntime, runtimeHandler, + r.sandboxSpec, ) if err != nil { return "", fmt.Errorf("failed to generate container %q spec: %w", r.containerID, err) @@ -612,6 +627,7 @@ func (c *criService) buildContainerSpec( extraMounts []*runtime.Mount, ociRuntime criconfig.Runtime, runtimeHandler *runtime.RuntimeHandler, + sandboxSpec *runtimespec.Spec, ) (_ *runtimespec.Spec, retErr error) { var ( specOpts []oci.SpecOpts @@ -627,12 +643,13 @@ func (c *criService) buildContainerSpec( case isLinux: // Generate container mounts. // No mounts are passed for other platforms. - linuxMounts := c.linuxContainerMounts(sandboxID, config) + linuxMounts := c.linuxContainerMounts(sandboxID, config, sandboxSpec) specOpts, err = c.buildLinuxSpec( id, sandboxID, sandboxPid, + sandboxSpec, containerName, imageName, config, @@ -680,6 +697,7 @@ func (c *criService) buildLinuxSpec( id string, sandboxID string, sandboxPid uint32, + sandboxSpec *runtimespec.Spec, containerName string, imageName string, config *runtime.ContainerConfig, @@ -922,7 +940,7 @@ func (c *criService) buildLinuxSpec( specOpts = append(specOpts, customopts.WithOOMScoreAdj(config, c.config.RestrictOOMScoreAdj), - customopts.WithPodNamespaces(securityContext, sandboxPid, targetPid, uids, gids), + customopts.WithPodNamespacePaths(securityContext, sandboxConfig.GetLinux().GetSecurityContext().GetNamespaceOptions(), podNamespacePaths(sandboxPid, targetPid, sandboxSpec), uids, gids), customopts.WithSupplementalGroups(supplementalGroups), ) specOpts = append( @@ -1084,9 +1102,94 @@ func (c *criService) buildDarwinSpec( return specOpts, nil } +// sandboxSpec loads the OCI spec the sandbox controller returned when the +// sandbox was started. It is nil when the controller returned none. +func (c *criService) sandboxSpec(ctx context.Context, sandboxID string) (*runtimespec.Spec, error) { + sandboxInfo, err := c.client.SandboxStore().Get(ctx, sandboxID) + if err != nil { + return nil, fmt.Errorf("failed to get sandbox %q from metadata store: %w", sandboxID, err) + } + if sandboxInfo.Spec == nil { + return nil, nil + } + var spec runtimespec.Spec + if err := typeurl.UnmarshalTo(sandboxInfo.Spec, &spec); err != nil { + return nil, fmt.Errorf("failed to unmarshal spec of sandbox %q: %w", sandboxID, err) + } + return &spec, nil +} + +// podNamespacePaths resolves the namespaces a container joins. +// +// A PID-backed sandbox (pause) is a member of every pod namespace, so the +// paths are derived from its pid exactly as before. A sandbox without a +// process holds the pod namespaces itself and lists their paths in the Linux +// section of the spec it returned on start; a namespace missing from that +// spec is one the sandbox does not hold, which WithPodNamespacePaths turns into +// the host namespace when the pod asked for it and into an error otherwise. +// +// A sandbox that reports no process and returns no spec (or none with a Linux +// section) is a pre-existing Sandbox API shim that describes neither (VM based +// shims ignore the paths CRI generates); those keep the historical pid derived +// construction so that their behavior does not change. +func podNamespacePaths(sandboxPid, targetPid uint32, sandboxSpec *runtimespec.Spec) customopts.PodNamespacePaths { + if sandboxPid != 0 || sandboxSpec == nil || sandboxSpec.Linux == nil { + return customopts.PodNamespacePathsFromPid(sandboxPid, targetPid) + } + var paths customopts.PodNamespacePaths + for _, ns := range sandboxSpec.Linux.Namespaces { + switch ns.Type { + case runtimespec.NetworkNamespace: + paths.Network = ns.Path + case runtimespec.IPCNamespace: + paths.IPC = ns.Path + case runtimespec.UTSNamespace: + paths.UTS = ns.Path + case runtimespec.PIDNamespace: + // A PID namespace entry without a path is the sandbox declaring + // that it holds none and that each container gets a new one. + paths.PID = ns.Path + paths.PrivatePID = ns.Path == "" + case runtimespec.UserNamespace: + paths.User = ns.Path + } + } + // A container targeting another container's PID namespace joins that + // container, not the sandbox. + if targetPid != sandboxPid { + paths.PID = customopts.GetPIDNamespace(targetPid) + } + return paths +} + +// sandboxSharedFile returns the host path of a pod shared file, i.e. a file +// the sandbox provides to every container of the pod at destination (for +// example /etc/hosts). The fixed location under the CRI sandbox directory +// wins when it exists; otherwise the mount the sandbox declared for that +// destination in its spec is used, which is how a sandbox implementation that +// owns the files, such as a Sandbox API shim, hands them over. The result is +// empty when neither exists. +func (c *criService) sandboxSharedFile(fixedPath, destination string, sandboxSpec *runtimespec.Spec) string { + if _, err := c.os.Stat(fixedPath); err == nil { + return fixedPath + } + if sandboxSpec == nil { + return "" + } + for _, m := range sandboxSpec.Mounts { + if m.Destination != destination || m.Source == "" { + continue + } + if _, err := c.os.Stat(m.Source); err == nil { + return m.Source + } + } + return "" +} + // linuxContainerMounts sets up necessary container system file mounts // including /dev/shm, /etc/hosts and /etc/resolv.conf. -func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.ContainerConfig) []*runtime.Mount { +func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.ContainerConfig, sandboxSpec *runtimespec.Spec) []*runtime.Mount { var mounts []*runtime.Mount securityContext := config.GetLinux().GetSecurityContext() var uidMappings, gidMappings []*runtime.IDMapping @@ -1101,8 +1204,7 @@ func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.Cont // do not mount this in that case. // TODO(random-liu): Remove the check and always mount this when // containerd 1.1 and 1.2 are deprecated. - hostpath := c.getSandboxHostname(sandboxID) - if _, err := c.os.Stat(hostpath); err == nil { + if hostpath := c.sandboxSharedFile(c.getSandboxHostname(sandboxID), etcHostname, sandboxSpec); hostpath != "" { mounts = append(mounts, &runtime.Mount{ ContainerPath: etcHostname, HostPath: hostpath, @@ -1115,10 +1217,9 @@ func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.Cont } if !isInCRIMounts(etcHosts, config.GetMounts()) { - hostpath := c.getSandboxHosts(sandboxID) // /etc/hosts could be delegated to remote sandbox controller. That file isn't required to be existed // in host side for some sandbox runtimes. Skip it if we don't need it. - if _, err := c.os.Stat(hostpath); err == nil { + if hostpath := c.sandboxSharedFile(c.getSandboxHosts(sandboxID), etcHosts, sandboxSpec); hostpath != "" { mounts = append(mounts, &runtime.Mount{ ContainerPath: etcHosts, HostPath: hostpath, @@ -1133,10 +1234,9 @@ func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.Cont // Mount sandbox resolv.config. // TODO: Need to figure out whether we should always mount it as read-only if !isInCRIMounts(resolvConfPath, config.GetMounts()) { - hostpath := c.getResolvPath(sandboxID) // The ownership of /etc/resolv.conf could be delegated to remote sandbox controller. That file isn't // required to be existed in host side for some sandbox runtimes. Skip it if we don't need it. - if _, err := c.os.Stat(hostpath); err == nil { + if hostpath := c.sandboxSharedFile(c.getResolvPath(sandboxID), resolvConfPath, sandboxSpec); hostpath != "" { mounts = append(mounts, &runtime.Mount{ ContainerPath: resolvConfPath, HostPath: hostpath, @@ -1152,6 +1252,8 @@ func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.Cont sandboxDevShm := c.getSandboxDevShm(sandboxID) if securityContext.GetNamespaceOptions().GetIpc() == runtime.NamespaceMode_NODE { sandboxDevShm = devShm + } else if source := c.sandboxSharedFile(sandboxDevShm, devShm, sandboxSpec); source != "" { + sandboxDevShm = source } // The ownership of /dev/shm could be delegated to remote sandbox controller. That file isn't required // to be existed in host side for some sandbox runtimes. Skip it if we don't need it. diff --git a/internal/cri/server/container_create_linux_test.go b/internal/cri/server/container_create_linux_test.go index aa23fd81ac..271a215e72 100644 --- a/internal/cri/server/container_create_linux_test.go +++ b/internal/cri/server/container_create_linux_test.go @@ -251,7 +251,7 @@ func TestContainerCapabilities(t *testing.T) { c.allCaps = allCaps containerConfig.Linux.SecurityContext.Capabilities = test.capability - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) if selinux.GetEnabled() { @@ -286,7 +286,7 @@ func TestContainerSpecTty(t *testing.T) { c := newTestCRIService() for _, tty := range []bool{true, false} { containerConfig.Tty = tty - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) assert.Equal(t, tty, spec.Process.Terminal) @@ -313,7 +313,7 @@ func TestContainerSpecDefaultPath(t *testing.T) { imageConfig.Env = append(imageConfig.Env, pathenv) expected = pathenv } - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) assert.Contains(t, spec.Process.Env, expected) @@ -330,7 +330,7 @@ func TestContainerSpecReadonlyRootfs(t *testing.T) { c := newTestCRIService() for _, readonly := range []bool{true, false} { containerConfig.Linux.SecurityContext.ReadonlyRootfs = readonly - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) assert.Equal(t, readonly, spec.Root.Readonly) @@ -364,7 +364,7 @@ func TestContainerSpecWithExtraMounts(t *testing.T) { Readonly: false, }, } - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, extraMounts, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, extraMounts, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) var mounts, sysMounts []runtimespec.Mount @@ -430,7 +430,7 @@ func TestContainerAndSandboxPrivileged(t *testing.T) { sandboxConfig.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ Privileged: test.sandboxPrivileged, } - _, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + _, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) if test.expectError { assert.Error(t, err) } else { @@ -470,7 +470,7 @@ func TestPrivilegedBindMount(t *testing.T) { containerConfig.Linux.SecurityContext.Privileged = test.privileged sandboxConfig.Linux.SecurityContext.Privileged = test.privileged - spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) if test.expectedSysFSRO { @@ -542,7 +542,7 @@ func TestCgroupNamespace(t *testing.T) { containerConfig.Linux.SecurityContext.Privileged = tt.privileged sandboxConfig.Linux.SecurityContext.Privileged = tt.privileged - spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) hasCgroupNS := false @@ -711,7 +711,7 @@ func TestPidNamespace(t *testing.T) { } { t.Run(test.desc, func(t *testing.T) { containerConfig.Linux.SecurityContext.NamespaceOptions = &runtime.NamespaceOption{Pid: test.pidNS} - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) assert.Contains(t, spec.Linux.Namespaces, test.expected) }) @@ -885,7 +885,7 @@ func TestUserNamespace(t *testing.T) { sandboxUserns = test.sandboxUserNS } sandboxConfig.Linux.SecurityContext.NamespaceOptions = &runtime.NamespaceOption{UsernsOptions: sandboxUserns} - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) if test.err { require.Error(t, err) @@ -915,7 +915,7 @@ func TestNoDefaultRunMount(t *testing.T) { ociRuntime := config.Runtime{} c := newTestCRIService() - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) for _, mount := range spec.Mounts { assert.NotEqual(t, "/run", mount.Destination) @@ -1008,7 +1008,7 @@ func TestMaskedAndReadonlyPaths(t *testing.T) { sandboxConfig.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ Privileged: test.privileged, } - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) if !test.privileged { // specCheck presumes an unprivileged container specCheck(t, testID, testSandboxID, testPid, spec) @@ -1060,7 +1060,7 @@ func TestHostname(t *testing.T) { sandboxConfig.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ NamespaceOptions: &runtime.NamespaceOption{Network: test.networkNs}, } - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) assert.Contains(t, spec.Process.Env, test.expectedEnv) @@ -1169,7 +1169,7 @@ additional-group-for-root:x:22222:root containerConfig.Linux.SecurityContext = test.securityContext imageConfig.User = test.imageConfigUser - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) spec.Root.Path = tempRootDir // simulating /etc/{passwd, group} @@ -1244,7 +1244,7 @@ func TestNonRootUserAndDevices(t *testing.T) { }, } - spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil) + spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil, nil) assert.NoError(t, err) assert.Equal(t, test.expectedDeviceUID, *spec.Linux.Devices[0].UID) @@ -1317,7 +1317,7 @@ func TestPrivilegedDevices(t *testing.T) { PrivilegedWithoutHostDevices: test.privilegedWithoutHostDevices, PrivilegedWithoutHostDevicesAllDevicesAllowed: test.privilegedWithoutHostDevicesAllDevicesAllowed, } - spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) hostDevicesRaw, err := oci.HostDevices() @@ -1372,7 +1372,7 @@ func TestBaseOCISpec(t *testing.T) { testPid := uint32(1234) containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData() - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) @@ -1700,7 +1700,7 @@ containerEdits: }, } { t.Run(test.description, func(t *testing.T) { - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) @@ -1816,7 +1816,7 @@ func TestUserNamespaceWithHostNetwork(t *testing.T) { }, } - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) require.NotNil(t, spec) diff --git a/internal/cri/server/container_create_sandboxspec_linux_test.go b/internal/cri/server/container_create_sandboxspec_linux_test.go new file mode 100644 index 0000000000..d71a5d17ef --- /dev/null +++ b/internal/cri/server/container_create_sandboxspec_linux_test.go @@ -0,0 +1,168 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package server + +import ( + "os" + "testing" + + runtimespec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + + customopts "github.com/containerd/containerd/v2/internal/cri/opts" + ostesting "github.com/containerd/containerd/v2/pkg/os/testing" +) + +// pathBackedSandboxSpec is what a sandbox without a process returns on start: +// the namespaces it holds and the pod shared files it owns. +func pathBackedSandboxSpec() *runtimespec.Spec { + return &runtimespec.Spec{ + Linux: &runtimespec.Linux{ + Namespaces: []runtimespec.LinuxNamespace{ + {Type: runtimespec.NetworkNamespace, Path: "/var/run/netns/cni-1"}, + {Type: runtimespec.IPCNamespace, Path: "/run/sandbox/ns/ipc"}, + {Type: runtimespec.UTSNamespace, Path: "/run/sandbox/ns/uts"}, + }, + }, + Mounts: []runtimespec.Mount{ + {Destination: "/etc/hostname", Type: "bind", Source: "/run/sandbox/hostname"}, + {Destination: "/etc/hosts", Type: "bind", Source: "/run/sandbox/hosts"}, + {Destination: "/etc/resolv.conf", Type: "bind", Source: "/run/sandbox/resolv.conf"}, + {Destination: "/dev/shm", Type: "bind", Source: "/run/sandbox/shm"}, + }, + } +} + +func TestPodNamespacePaths(t *testing.T) { + spec := pathBackedSandboxSpec() + for name, tc := range map[string]struct { + sandboxPid, targetPid uint32 + spec *runtimespec.Spec + expected customopts.PodNamespacePaths + }{ + "a sandbox process wins over the spec": { + sandboxPid: 42, targetPid: 42, spec: spec, + expected: customopts.PodNamespacePathsFromPid(42, 42), + }, + "no process and no spec is a legacy sandbox api shim": { + expected: customopts.PodNamespacePathsFromPid(0, 0), + }, + "no process and a spec without a linux section is legacy too": { + spec: &runtimespec.Spec{}, + expected: customopts.PodNamespacePathsFromPid(0, 0), + }, + "no process and a spec with namespaces is path backed": { + spec: spec, + expected: customopts.PodNamespacePaths{ + Network: "/var/run/netns/cni-1", + IPC: "/run/sandbox/ns/ipc", + UTS: "/run/sandbox/ns/uts", + }, + }, + "a path backed sandbox in the host namespaces provides none": { + spec: &runtimespec.Spec{Linux: &runtimespec.Linux{}}, + expected: customopts.PodNamespacePaths{}, + }, + "a pid namespace entry without a path declares private pid namespaces": { + spec: &runtimespec.Spec{Linux: &runtimespec.Linux{Namespaces: []runtimespec.LinuxNamespace{ + {Type: runtimespec.NetworkNamespace, Path: "/var/run/netns/cni-1"}, + {Type: runtimespec.PIDNamespace}, + }}}, + expected: customopts.PodNamespacePaths{Network: "/var/run/netns/cni-1", PrivatePID: true}, + }, + "a target container pid namespace is joined by pid": { + targetPid: 7, spec: spec, + expected: customopts.PodNamespacePaths{ + Network: "/var/run/netns/cni-1", + IPC: "/run/sandbox/ns/ipc", + UTS: "/run/sandbox/ns/uts", + PID: "/proc/7/ns/pid", + }, + }, + } { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, podNamespacePaths(tc.sandboxPid, tc.targetPid, tc.spec)) + }) + } +} + +func TestLinuxContainerMountsFromSandboxSpec(t *testing.T) { + const testSandboxID = "test-sandbox-id" + config := &runtime.ContainerConfig{ + Metadata: &runtime.ContainerMetadata{Name: "test-name", Attempt: 1}, + Linux: &runtime.LinuxContainerConfig{ + SecurityContext: &runtime.LinuxContainerSecurityContext{}, + }, + } + exists := func(paths ...string) func(string) (os.FileInfo, error) { + set := make(map[string]struct{}, len(paths)) + for _, p := range paths { + set[p] = struct{}{} + } + return func(p string) (os.FileInfo, error) { + if _, ok := set[p]; ok { + return nil, nil + } + return nil, os.ErrNotExist + } + } + sharedMount := func(containerPath, hostPath string) *runtime.Mount { + return &runtime.Mount{ContainerPath: containerPath, HostPath: hostPath, SelinuxRelabel: true} + } + + t.Run("the sandbox spec provides the pod shared files", func(t *testing.T) { + c := newTestCRIService() + c.os.(*ostesting.FakeOS).StatFn = exists("/run/sandbox/hostname", "/run/sandbox/hosts", "/run/sandbox/resolv.conf", "/run/sandbox/shm") + assert.Equal(t, []*runtime.Mount{ + sharedMount(etcHostname, "/run/sandbox/hostname"), + sharedMount(etcHosts, "/run/sandbox/hosts"), + sharedMount(resolvConfPath, "/run/sandbox/resolv.conf"), + sharedMount(devShm, "/run/sandbox/shm"), + }, c.linuxContainerMounts(testSandboxID, config, pathBackedSandboxSpec())) + }) + + t.Run("the fixed sandbox directory wins over the spec", func(t *testing.T) { + c := newTestCRIService() + c.os.(*ostesting.FakeOS).StatFn = exists(c.getSandboxHosts(testSandboxID), "/run/sandbox/hosts") + assert.Equal(t, []*runtime.Mount{ + sharedMount(etcHosts, c.getSandboxHosts(testSandboxID)), + }, c.linuxContainerMounts(testSandboxID, config, pathBackedSandboxSpec())) + }) + + t.Run("a spec source that does not exist is not mounted", func(t *testing.T) { + c := newTestCRIService() + c.os.(*ostesting.FakeOS).StatFn = exists() + assert.Empty(t, c.linuxContainerMounts(testSandboxID, config, pathBackedSandboxSpec())) + }) + + t.Run("host ipc uses the host shm", func(t *testing.T) { + c := newTestCRIService() + c.os.(*ostesting.FakeOS).StatFn = exists(devShm, "/run/sandbox/shm") + hostIPC := &runtime.ContainerConfig{ + Metadata: config.Metadata, + Linux: &runtime.LinuxContainerConfig{ + SecurityContext: &runtime.LinuxContainerSecurityContext{ + NamespaceOptions: &runtime.NamespaceOption{Ipc: runtime.NamespaceMode_NODE}, + }, + }, + } + assert.Equal(t, []*runtime.Mount{ + {ContainerPath: devShm, HostPath: devShm, SelinuxRelabel: false}, + }, c.linuxContainerMounts(testSandboxID, hostIPC, pathBackedSandboxSpec())) + }) +} diff --git a/internal/cri/server/container_create_test.go b/internal/cri/server/container_create_test.go index 227b78aeac..ff7cb0aea1 100644 --- a/internal/cri/server/container_create_test.go +++ b/internal/cri/server/container_create_test.go @@ -70,7 +70,7 @@ func TestGeneralContainerSpec(t *testing.T) { c := newTestCRIService() testSandboxID := "sandbox-id" testContainerName := "container-name" - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) } @@ -146,7 +146,7 @@ func TestPodAnnotationPassthroughContainerSpec(t *testing.T) { PodAnnotations: test.podAnnotations, } spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, - containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) assert.NotNil(t, spec) specCheck(t, testID, testSandboxID, testPid, spec) @@ -508,7 +508,7 @@ func TestContainerAnnotationPassthroughContainerSpec(t *testing.T) { ContainerAnnotations: test.containerAnnotations, } spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, - containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) assert.NotNil(t, spec) specCheck(t, testID, testSandboxID, testPid, spec) @@ -759,7 +759,7 @@ func TestLinuxContainerMounts(t *testing.T) { } c := newTestCRIService() c.os.(*ostesting.FakeOS).StatFn = test.statFn - mounts := c.linuxContainerMounts(testSandboxID, config) + mounts := c.linuxContainerMounts(testSandboxID, config, nil) assert.Equal(t, test.expectedMounts, mounts, test.desc) }) } diff --git a/internal/cri/server/container_create_windows_test.go b/internal/cri/server/container_create_windows_test.go index 2a12d16237..5cfe1401eb 100644 --- a/internal/cri/server/container_create_windows_test.go +++ b/internal/cri/server/container_create_windows_test.go @@ -165,7 +165,7 @@ func TestContainerWindowsNetworkNamespace(t *testing.T) { c := newTestCRIService() containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData() - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil, nil) assert.NoError(t, err) assert.NotNil(t, spec) specCheck(t, testID, testSandboxID, testPid, spec) @@ -187,7 +187,7 @@ func TestMountCleanPath(t *testing.T) { ContainerPath: "c:/test/container-path", HostPath: "c:/test/host-path", }) - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil, nil) assert.NoError(t, err) assert.NotNil(t, spec) specCheck(t, testID, testSandboxID, testPid, spec) @@ -207,7 +207,7 @@ func TestMountNamedPipe(t *testing.T) { ContainerPath: `\\.\pipe\foo`, HostPath: `\\.\pipe\foo`, }) - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil, nil) assert.NoError(t, err) assert.NotNil(t, spec) specCheck(t, testID, testSandboxID, testPid, spec) @@ -258,7 +258,7 @@ func TestHostProcessRequirements(t *testing.T) { sandboxConfig.Windows.SecurityContext = &runtime.WindowsSandboxSecurityContext{ HostProcess: test.sandboxHostProcess, } - _, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + _, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) if test.expectError { assert.Error(t, err) } else { @@ -355,7 +355,7 @@ func TestEntrypointAndCmdForArgsEscaped(t *testing.T) { Args: test.args, Windows: &runtime.WindowsContainerConfig{}, } - runtimeSpec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil) + runtimeSpec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil, nil) assert.NoError(t, err) assert.NotNil(t, runtimeSpec) From 423f5e745db3fa83f26617fcb7cc243a0e435323 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Tue, 8 Sep 2026 19:32:33 -0700 Subject: [PATCH 06/13] cmd: add containerd-shim-sandboxed-runc-v2 An experimental runc shim that implements the Sandbox API itself, so that a pod needs no pause container and no pause image. It serves the io.containerd.runc.v2 runtime type from a separate binary; a runtime handler selects it with runtime_path and pairs it with the shim sandboxer: [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] runtime_type = "io.containerd.runc.v2" runtime_path = "/usr/local/bin/containerd-shim-sandboxed-runc-v2" sandboxer = "shim" disable_pause_image_pull = true The shim is the sandbox. On CreateSandbox a locked thread enters the network namespace CRI created, unshares the pod IPC and UTS namespaces, sets the hostname and the pod sysctls from inside them, bind-mounts its namespace files under the bundle and moves back out; the pins keep the namespaces alive the way CRI keeps the network namespace alive. The shim writes the pod shared files and mounts the shm tmpfs under its bundle and returns them as mounts of the sandbox spec, together with the namespace paths. StartSandbox reports pid 0. The containers of the pod run through the task service of containerd-shim-runc-v2, wrapped only so that the task Shutdown RPC, which containerd sends after every task deletion, does not end the shim while the pod is alive. StopSandbox and "shim delete" detach the pins and the shm and remove the files without relying on any state; ShutdownSandbox ends the shim. The runtime info is the runc shim's minus the user namespace, which the shim does not support yet. Not supported yet: a shared pod PID namespace (the containers of such a pod get their own PID namespaces, with a warning), user namespaces, user.* sysctls, SELinux labels, cgroup v1 and rootless containerd. The shim decodes the CRI PodSandboxConfig it receives as options; a containerd owned configuration message is left as a TODO. The binary is built and installed with the other Linux commands. Signed-off-by: Maksym Pavlenko --- Makefile | 4 + Makefile.linux | 1 + cmd/containerd-shim-sandboxed-runc-v2/main.go | 36 ++ .../manager.go | 296 ++++++++++++++ .../manager_test.go | 61 +++ .../sandbox/bundle.go | 145 +++++++ .../sandbox/bundle_test.go | 128 ++++++ .../sandbox/config.go | 275 +++++++++++++ .../sandbox/config_test.go | 360 +++++++++++++++++ .../sandbox/doc.go | 100 +++++ .../sandbox/namespaces.go | 233 +++++++++++ .../sandbox/namespaces_test.go | 152 ++++++++ .../sandbox/service.go | 366 ++++++++++++++++++ .../sandbox/service_test.go | 113 ++++++ cmd/containerd-shim-sandboxed-runc-v2/task.go | 100 +++++ .../task_test.go | 53 +++ 16 files changed, 2423 insertions(+) create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/main.go create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/manager.go create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/manager_test.go create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle.go create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle_test.go create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/sandbox/config.go create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/sandbox/config_test.go create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/sandbox/doc.go create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/sandbox/namespaces.go create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/sandbox/namespaces_test.go create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/sandbox/service.go create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/sandbox/service_test.go create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/task.go create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/task_test.go diff --git a/Makefile b/Makefile index a39c97e0f8..f6e0c71993 100644 --- a/Makefile +++ b/Makefile @@ -272,6 +272,10 @@ bin/containerd-shim-runc-v2: cmd/containerd-shim-runc-v2 FORCE # set !cgo and om @echo "$(WHALE) $@" CGO_ENABLED=${SHIM_CGO_ENABLED} $(GO) build ${GO_BUILD_FLAGS} -o $@ ${SHIM_GO_LDFLAGS} ${SHIM_GO_TAGS} ./cmd/containerd-shim-runc-v2 +bin/containerd-shim-sandboxed-runc-v2: cmd/containerd-shim-sandboxed-runc-v2 FORCE # experimental pause-less sandbox shim, static like containerd-shim-runc-v2 + @echo "$(WHALE) $@" + CGO_ENABLED=${SHIM_CGO_ENABLED} $(GO) build ${GO_BUILD_FLAGS} -o $@ ${SHIM_GO_LDFLAGS} ${SHIM_GO_TAGS} ./cmd/containerd-shim-sandboxed-runc-v2 + binaries: $(BINARIES) ## build binaries @echo "$(WHALE) $@" diff --git a/Makefile.linux b/Makefile.linux index 6f44a29336..8b9750aebe 100644 --- a/Makefile.linux +++ b/Makefile.linux @@ -17,6 +17,7 @@ WHALE="+" ONI="-" COMMANDS += containerd-shim-runc-v2 +COMMANDS += containerd-shim-sandboxed-runc-v2 # check GOOS for cross compile builds ifeq ($(GOOS),linux) diff --git a/cmd/containerd-shim-sandboxed-runc-v2/main.go b/cmd/containerd-shim-sandboxed-runc-v2/main.go new file mode 100644 index 0000000000..00a963ea9a --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/main.go @@ -0,0 +1,36 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// containerd-shim-sandboxed-runc-v2 is an experimental runc shim that +// implements the Sandbox API itself, so that a pod needs no pause container: +// the shim holds the pod namespaces and owns the pod shared files, and runs +// the containers of the pod with the same runc task service as +// containerd-shim-runc-v2. It serves the io.containerd.runc.v2 runtime type; +// a runtime handler selects it with runtime_path and sandboxer = "shim". See +// the sandbox package for details on the design. +package main + +import ( + "context" + + "github.com/containerd/containerd/v2/pkg/shim" +) + +func main() { + shim.RunShim(context.Background(), newManager()) +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/manager.go b/cmd/containerd-shim-sandboxed-runc-v2/manager.go new file mode 100644 index 0000000000..114999311d --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/manager.go @@ -0,0 +1,296 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + goruntime "runtime" + "slices" + "syscall" + + "github.com/containerd/cgroups/v3" + cgroupsv2 "github.com/containerd/cgroups/v3/cgroup2" + bootapi "github.com/containerd/containerd/api/runtime/bootstrap/v1" + "github.com/containerd/containerd/api/types" + "github.com/containerd/containerd/api/types/runc/options" + "github.com/containerd/errdefs" + "github.com/containerd/typeurl/v2" + "github.com/opencontainers/runtime-spec/specs-go/features" + + legacymanager "github.com/containerd/containerd/v2/cmd/containerd-shim-runc-v2/manager" + "github.com/containerd/containerd/v2/cmd/containerd-shim-sandboxed-runc-v2/sandbox" + "github.com/containerd/containerd/v2/defaults" + "github.com/containerd/containerd/v2/pkg/namespaces" + "github.com/containerd/containerd/v2/pkg/schedcore" + "github.com/containerd/containerd/v2/pkg/shim" + "github.com/containerd/containerd/v2/plugins" +) + +// manager is a fork of the containerd-shim-runc-v2 manager for the sandbox +// flow: "start" runs once per pod, for the sandbox, keyed by the sandbox id +// (containers join the shim through the sandbox endpoint, so there is no +// grouping by OCI annotations); "delete" also cleans up a sandbox bundle; and +// "-info" hides the user namespace feature. +// +// TODO(mxpv): consolidate with the containerd-shim-runc-v2 manager. +type manager struct { + // legacy is the containerd-shim-runc-v2 manager. It serves the runtime + // info (the same runtime type, the same runc, the same features) and the + // cleanup of the task bundles joined to this shim. + legacy shim.Shim +} + +func newManager() shim.Shim { + return &manager{legacy: legacymanager.NewShimManager(plugins.RuntimeRuncV2)} +} + +// Name is the runtime type served: containerd reaches this binary through +// runtime_path, not through a runtime type of its own. +func (m *manager) Name() string { + return m.legacy.Name() +} + +func newCommand(ctx context.Context, id, containerdAddress string, debug bool) (*exec.Cmd, error) { + ns, err := namespaces.NamespaceRequired(ctx) + if err != nil { + return nil, err + } + self, err := os.Executable() + if err != nil { + return nil, err + } + cwd, err := os.Getwd() + if err != nil { + return nil, err + } + args := []string{ + "-namespace", ns, + "-id", id, + "-address", containerdAddress, + } + if debug { + args = append(args, "-debug") + } + cmd := exec.Command(self, args...) + cmd.Dir = cwd + cmd.Env = append(os.Environ(), "GOMAXPROCS=4") + cmd.Env = append(cmd.Env, "OTEL_SERVICE_NAME=containerd-shim-"+id) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, + } + return cmd, nil +} + +type shimSocket struct { + addr string + s *net.UnixListener + f *os.File +} + +func (s *shimSocket) Close() { + if s.s != nil { + s.s.Close() + } + if s.f != nil { + s.f.Close() + } + _ = shim.RemoveSocket(s.addr) +} + +func newShimSocket(ctx context.Context, root, path, id string, debug bool) (*shimSocket, error) { + address, err := shim.CreateSocketAddress(ctx, root, path, id, debug) + if err != nil { + return nil, err + } + socket, err := shim.NewSocket(address) + if err != nil { + // A socket in use that accepts connections belongs to a shim that is + // already running for this sandbox; a stale one is replaced. + if !shim.SocketEaddrinuse(err) { + return nil, fmt.Errorf("create new shim socket: %w", err) + } + if !debug && shim.CanConnect(address) { + return &shimSocket{addr: address}, errdefs.ErrAlreadyExists + } + if err := shim.RemoveSocket(address); err != nil { + return nil, fmt.Errorf("remove pre-existing socket: %w", err) + } + if socket, err = shim.NewSocket(address); err != nil { + return nil, fmt.Errorf("try create new shim socket 2x: %w", err) + } + } + s := &shimSocket{ + addr: address, + s: socket, + } + f, err := socket.File() + if err != nil { + s.Close() + return nil, err + } + s.f = f + return s, nil +} + +// Start starts the shim process for a sandbox bundle. containerd calls it once +// per sandbox; the containers of the pod join the running shim through the +// bootstrap protocol without invoking "start" again. +func (m *manager) Start(ctx context.Context, opts *bootapi.BootstrapParams) (_ *bootapi.BootstrapResult, retErr error) { + if cgroups.Mode() != cgroups.Unified { + return nil, fmt.Errorf("%s requires cgroup v2 (the unified hierarchy) and this host runs cgroup v1: %w", sandbox.BinaryName, errdefs.ErrNotImplemented) + } + + var params bootapi.BootstrapResult + params.Version = 3 + params.Protocol = "ttrpc" + + id := opts.GetInstanceID() + debugLog := opts.GetLogLevel() <= bootapi.LogLevel_LOG_LEVEL_DEBUG + + cmd, err := newCommand(ctx, id, opts.GetContainerdGrpcAddress(), debugLog) + if err != nil { + return nil, err + } + + var sockets []*shimSocket + defer func() { + if retErr != nil { + for _, s := range sockets { + s.Close() + } + } + }() + + socketDir := opts.GetSocketDir() + if socketDir == "" { + socketDir = filepath.Join(defaults.DefaultStateDir, "s") + } + // The socket is keyed by the sandbox id alone. + s, err := newShimSocket(ctx, socketDir, opts.GetContainerdGrpcAddress(), id, false) + if err != nil { + if errdefs.IsAlreadyExists(err) { + params.Address = s.addr + return ¶ms, nil + } + return nil, err + } + sockets = append(sockets, s) + cmd.ExtraFiles = append(cmd.ExtraFiles, s.f) + + if debugLog { + s, err = newShimSocket(ctx, socketDir, opts.GetContainerdGrpcAddress(), id, true) + if err != nil { + return nil, err + } + sockets = append(sockets, s) + cmd.ExtraFiles = append(cmd.ExtraFiles, s.f) + } + + // The shim is spawned from a locked thread so that a core scheduling + // cookie created here is inherited by it; the thread is released on + // every path. + goruntime.LockOSThread() + err = func() error { + if os.Getenv("SCHED_CORE") != "" { + if err := schedcore.Create(schedcore.ProcessGroup); err != nil { + return fmt.Errorf("enable sched core support: %w", err) + } + } + return cmd.Start() + }() + goruntime.UnlockOSThread() + if err != nil { + return nil, err + } + + defer func() { + if retErr != nil { + cmd.Process.Kill() + } + }() + // make sure to wait after start + go cmd.Wait() + + var runcOpts options.Options + if found, err := opts.FindExtension(&runcOpts); err != nil { + return nil, fmt.Errorf("failed to fetch runc options: %w", err) + } else if found { + if shimCgroup := runcOpts.GetShimCgroup(); shimCgroup != "" { + cg, err := cgroupsv2.Load(shimCgroup) + if err != nil { + return nil, fmt.Errorf("failed to load cgroup %s: %w", shimCgroup, err) + } + if err := cg.AddProc(uint64(cmd.Process.Pid)); err != nil { + return nil, fmt.Errorf("failed to join cgroup %s: %w", shimCgroup, err) + } + } + } + + if err := shim.AdjustOOMScore(cmd.Process.Pid); err != nil { + return nil, fmt.Errorf("failed to adjust OOM score for shim: %w", err) + } + + params.Address = sockets[0].addr + return ¶ms, nil +} + +// Stop is "shim delete". containerd runs it in a bundle once the shim that +// served the bundle is gone: in the sandbox bundle, and in the bundle of every +// container task that joined the shim. Nothing needs telling apart: the runc +// task cleanup of containerd-shim-runc-v2 (force-delete the runc container of +// that id, detach its rootfs) does nothing in the sandbox bundle, and the +// sandbox cleanup (detach every mount under the bundle, remove the pod files) +// does nothing in a task bundle, so both run every time. +func (m *manager) Stop(ctx context.Context, id string) (shim.StopStatus, error) { + status, err := m.legacy.Stop(ctx, id) + cwd, cwdErr := os.Getwd() + if cwdErr != nil { + return status, errors.Join(err, cwdErr) + } + return status, errors.Join(err, sandbox.Cleanup(ctx, filepath.Join(filepath.Dir(cwd), id))) +} + +// Info returns the runtime info of containerd-shim-runc-v2: the same runtime +// type, and the features of the same runc minus the user namespace. CRI +// advertises user namespace support for a handler from that feature, and this +// shim rejects user namespaced pods for now. +func (m *manager) Info(ctx context.Context, optionsR io.Reader) (*types.RuntimeInfo, error) { + info, err := m.legacy.Info(ctx, optionsR) + if err != nil || info.GetFeatures() == nil { + return info, err + } + var feat features.Features + if err := typeurl.UnmarshalTo(info.Features, &feat); err != nil { + return nil, fmt.Errorf("failed to decode the runc features: %w", err) + } + if feat.Linux != nil { + feat.Linux.Namespaces = slices.DeleteFunc(feat.Linux.Namespaces, func(ns string) bool { return ns == "user" }) + } + if info.Features, err = typeurl.MarshalAnyToProto(&feat); err != nil { + return nil, fmt.Errorf("failed to encode the runc features: %w", err) + } + return info, nil +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/manager_test.go b/cmd/containerd-shim-sandboxed-runc-v2/manager_test.go new file mode 100644 index 0000000000..db3b171c8e --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/manager_test.go @@ -0,0 +1,61 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/containerd/containerd/v2/pkg/namespaces" + "github.com/containerd/containerd/v2/plugins" +) + +func TestManagerName(t *testing.T) { + assert.Equal(t, plugins.RuntimeRuncV2, newManager().Name()) +} + +func TestStopCleansUpTheBundle(t *testing.T) { + // "shim delete" in a sandbox bundle removes what the sandbox created and + // leaves what containerd created. The runc task cleanup that runs first + // finds no container there and only logs about it. + root := t.TempDir() + const id = "sandbox-id" + bundle := filepath.Join(root, id) + require.NoError(t, os.MkdirAll(filepath.Join(bundle, "ns"), 0o700)) + require.NoError(t, os.Mkdir(filepath.Join(bundle, "rootfs"), 0o700)) + for _, f := range []string{"hostname", "hosts", "resolv.conf", filepath.Join("ns", "ipc"), filepath.Join("ns", "uts")} { + require.NoError(t, os.WriteFile(filepath.Join(bundle, f), nil, 0o600)) + } + t.Chdir(bundle) + + _, err := newManager().Stop(namespaces.WithNamespace(context.Background(), "test"), id) + require.NoError(t, err) + + for _, f := range []string{"hostname", "hosts", "resolv.conf", "ns"} { + _, err := os.Stat(filepath.Join(bundle, f)) + assert.True(t, os.IsNotExist(err), "%s should be gone", f) + } + _, err = os.Stat(filepath.Join(bundle, "rootfs")) + assert.NoError(t, err, "what containerd created is left alone") +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle.go new file mode 100644 index 0000000000..c2373a1401 --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle.go @@ -0,0 +1,145 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/moby/sys/mountinfo" + specs "github.com/opencontainers/runtime-spec/specs-go" + "golang.org/x/sys/unix" + + "github.com/containerd/containerd/v2/core/mount" + "github.com/containerd/containerd/v2/internal/cri/sandboxfiles" +) + +// Everything the sandbox creates lives under the shim bundle at fixed paths +// (see the package doc): +// +// /ns/{ipc,uts} namespace pins +// /hostname the pod hostname +// /hosts a copy of the host /etc/hosts +// /resolv.conf the pod DNS configuration +// /shm the pod /dev/shm tmpfs + +// setupFiles creates the pod shared files and returns the mounts that expose +// them to the containers of the pod; CRI picks the sources up from the +// sandbox spec. +func setupFiles(bundle string, cfg *Config) ([]specs.Mount, error) { + var mounts []specs.Mount + bind := func(destination, source string) { + mounts = append(mounts, specs.Mount{ + Destination: destination, + Type: "bind", + Source: source, + Options: []string{"rbind"}, + }) + } + + hostname := filepath.Join(bundle, sandboxfiles.HostnameFile) + if err := os.WriteFile(hostname, sandboxfiles.HostnameContent(cfg.Hostname), 0o644); err != nil { + return nil, fmt.Errorf("failed to write %s: %w", hostname, err) + } + bind(sandboxfiles.EtcHostname, hostname) + + hosts := filepath.Join(bundle, sandboxfiles.HostsFile) + if err := copyFile(sandboxfiles.EtcHosts, hosts); err != nil { + return nil, fmt.Errorf("failed to create %s: %w", hosts, err) + } + bind(sandboxfiles.EtcHosts, hosts) + + resolvConf := filepath.Join(bundle, sandboxfiles.ResolvConfFile) + if cfg.DNS != nil { + content := sandboxfiles.ResolvConfContent(cfg.DNS.GetServers(), cfg.DNS.GetSearches(), cfg.DNS.GetOptions()) + if err := os.WriteFile(resolvConf, content, 0o644); err != nil { + return nil, fmt.Errorf("failed to write %s: %w", resolvConf, err) + } + } else { + // No DNS config means the host resolver configuration, as for pause. + if err := copyFile(sandboxfiles.ResolvConfPath, resolvConf); err != nil { + return nil, fmt.Errorf("failed to create %s: %w", resolvConf, err) + } + } + bind(sandboxfiles.ResolvConfPath, resolvConf) + + if !cfg.HostIPC { + shm := filepath.Join(bundle, sandboxfiles.ShmDir) + if err := os.Mkdir(shm, 0o700); err != nil && !errors.Is(err, fs.ErrExist) { + return nil, fmt.Errorf("failed to create %s: %w", shm, err) + } + if err := unix.Mount("shm", shm, "tmpfs", unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV, sandboxfiles.ShmMountData(cfg.ShmSize)); err != nil { + return nil, fmt.Errorf("failed to mount the pod shm at %s: %w", shm, err) + } + bind(sandboxfiles.DevShm, shm) + } + return mounts, nil +} + +func copyFile(src, dst string) error { + b, err := os.ReadFile(src) + if err != nil { + return err + } + return os.WriteFile(dst, b, 0o644) +} + +// Cleanup removes everything the sandbox shim created under bundle: it +// detaches the mounts at their fixed paths (the namespace pins and the shm +// tmpfs) and removes the pin and shared files. It is idempotent and relies on +// no state, so it serves create failure, StopSandbox and "shim delete" alike; +// a mount left behind would keep containerd from deleting the bundle. +func Cleanup(ctx context.Context, bundle string) error { + var errs []error + for _, p := range []string{ + filepath.Join(bundle, pinDir, ipcPin), + filepath.Join(bundle, pinDir, utsPin), + filepath.Join(bundle, sandboxfiles.ShmDir), + } { + // Only a mount point is unmounted: umount(2) on a plain file needs + // the same privilege as on a mount, and answers EPERM before EINVAL. + mounted, err := mountinfo.Mounted(p) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + errs = append(errs, fmt.Errorf("failed to check %s: %w", p, err)) + } + continue + } + if !mounted { + continue + } + if err := mount.UnmountAll(p, unix.MNT_DETACH); err != nil { + errs = append(errs, fmt.Errorf("failed to detach %s: %w", p, err)) + } + } + if len(errs) > 0 { + // Nothing is removed while one of the mounts is still in place. + return errors.Join(errs...) + } + for _, name := range []string{pinDir, sandboxfiles.ShmDir, sandboxfiles.HostnameFile, sandboxfiles.HostsFile, sandboxfiles.ResolvConfFile} { + if err := os.RemoveAll(filepath.Join(bundle, name)); err != nil { + errs = append(errs, fmt.Errorf("failed to remove %s: %w", name, err)) + } + } + return errors.Join(errs...) +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle_test.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle_test.go new file mode 100644 index 0000000000..bb5dc2e06c --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle_test.go @@ -0,0 +1,128 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "os" + "path/filepath" + "testing" + + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" + criruntime "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/containerd/containerd/v2/pkg/testutil" +) + +func TestSetupFiles(t *testing.T) { + testutil.RequiresRoot(t) + ctx := context.Background() + bundle := t.TempDir() + t.Cleanup(func() { _ = Cleanup(ctx, bundle) }) + + cfg := &Config{ + ID: testID, + Hostname: "pod-hostname", + DNS: &criruntime.DNSConfig{ + Servers: []string{"10.0.0.10"}, + Searches: []string{"svc.cluster.local"}, + Options: []string{"ndots:5"}, + }, + ShmSize: 1 << 20, + } + mounts, err := setupFiles(bundle, cfg) + require.NoError(t, err) + + byDestination := make(map[string]specs.Mount) + for _, m := range mounts { + byDestination[m.Destination] = m + } + require.Len(t, byDestination, 4) + + content, err := os.ReadFile(byDestination["/etc/hostname"].Source) + require.NoError(t, err) + assert.Equal(t, "pod-hostname\n", string(content)) + + content, err = os.ReadFile(byDestination["/etc/resolv.conf"].Source) + require.NoError(t, err) + assert.Equal(t, "search svc.cluster.local\nnameserver 10.0.0.10\noptions ndots:5\n", string(content)) + + hostHosts, err := os.ReadFile("/etc/hosts") + require.NoError(t, err) + content, err = os.ReadFile(byDestination["/etc/hosts"].Source) + require.NoError(t, err) + assert.Equal(t, string(hostHosts), string(content), "the pod hosts file starts as a copy of the host one") + + var st unix.Statfs_t + require.NoError(t, unix.Statfs(byDestination["/dev/shm"].Source, &st)) + assert.EqualValues(t, unix.TMPFS_MAGIC, st.Type, "the pod shm is a tmpfs") + assert.EqualValues(t, 1<<20, st.Blocks*uint64(st.Bsize), "sized as requested") + + require.NoError(t, Cleanup(ctx, bundle)) + assertNoMountsUnder(t, bundle) + entries, err := os.ReadDir(bundle) + require.NoError(t, err) + assert.Empty(t, entries) + + // With the host IPC namespace the pod uses the host /dev/shm: no tmpfs. + cfg.HostIPC = true + cfg.DNS = nil + mounts, err = setupFiles(bundle, cfg) + require.NoError(t, err) + require.Len(t, mounts, 3) + for _, m := range mounts { + assert.NotEqual(t, "/dev/shm", m.Destination) + } + hostResolv, err := os.ReadFile("/etc/resolv.conf") + require.NoError(t, err) + content, err = os.ReadFile(filepath.Join(bundle, "resolv.conf")) + require.NoError(t, err) + assert.Equal(t, string(hostResolv), string(content), "no DNS config means the host resolver") +} + +func TestCleanupWithoutMounts(t *testing.T) { + ctx := context.Background() + bundle := t.TempDir() + + // Leftovers of a sandbox whose mounts are already gone (or that crashed + // before mounting anything) are removed, and nothing else is touched. + require.NoError(t, os.Mkdir(filepath.Join(bundle, pinDir), 0o700)) + require.NoError(t, os.Mkdir(filepath.Join(bundle, "shm"), 0o700)) + require.NoError(t, os.Mkdir(filepath.Join(bundle, "rootfs"), 0o700)) + for _, f := range []string{"hostname", "hosts", "resolv.conf", "ns/ipc", "ns/uts", "address"} { + require.NoError(t, os.WriteFile(filepath.Join(bundle, f), []byte("x"), 0o600)) + } + + require.NoError(t, Cleanup(ctx, bundle)) + + entries, err := os.ReadDir(bundle) + require.NoError(t, err) + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + assert.ElementsMatch(t, []string{"address", "rootfs"}, names, "only what containerd owns is left") + + // Idempotent, including on a bundle that no longer exists. + require.NoError(t, Cleanup(ctx, bundle)) + require.NoError(t, Cleanup(ctx, filepath.Join(bundle, "does-not-exist"))) +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config.go new file mode 100644 index 0000000000..94e5c0af57 --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config.go @@ -0,0 +1,275 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "fmt" + "os" + "strings" + + api "github.com/containerd/containerd/api/runtime/sandbox/v1" + "github.com/containerd/errdefs" + "github.com/containerd/typeurl/v2" + specs "github.com/opencontainers/runtime-spec/specs-go" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/containerd/containerd/v2/internal/cri/sandboxfiles" +) + +// Config is what the sandbox shim needs to know about a pod. It is derived +// from the CRI PodSandboxConfig that CRI passes as the sandbox options. +type Config struct { + // ID is the sandbox id. + ID string + // Hostname is the pod hostname, the node hostname when the pod sets none. + Hostname string + // HostNetwork means the pod uses the host network and UTS namespaces; the + // sandbox then holds neither and NetNSPath is empty. + HostNetwork bool + // HostIPC means the pod uses the host IPC namespace and the host /dev/shm. + HostIPC bool + // HostPID means the containers of the pod use the host PID namespace. + HostPID bool + // SharedPID means the pod asked for one PID namespace shared by its + // containers, which this shim cannot provide yet (see buildSpec). + SharedPID bool + // NetNSPath is the network namespace CRI created for the pod. + NetNSPath string + // Sysctls is the effective set of namespaced sysctls to apply. + Sysctls map[string]string + // CgroupParent is the pod cgroup as a cgroup v2 group path, used for + // metrics. kubelet spells it that way with both cgroup drivers: the + // systemd driver hands over the expanded slice path, which CRI shortens + // to its base for runc. + CgroupParent string + // DNS is the pod DNS configuration; nil means the host resolv.conf. + DNS *runtime.DNSConfig + // ShmSize is the size of the pod /dev/shm tmpfs. + ShmSize int64 + // Annotations are the annotations of the synthesized sandbox spec. + Annotations map[string]string +} + +// ConfigFromRequest derives the sandbox configuration from a create request. +// It rejects what this shim does not support yet with ErrNotImplemented so the +// error is explicit rather than a pod that silently lacks isolation. +func ConfigFromRequest(r *api.CreateSandboxRequest) (*Config, error) { + id := r.GetSandboxID() + if r.GetOptions() == nil { + return nil, fmt.Errorf("sandbox %q: no options; %s expects the CRI PodSandboxConfig: %w", id, BinaryName, errdefs.ErrInvalidArgument) + } + // TODO: define a containerd-owned sandbox configuration message for the + // handoff (hostname, namespace modes, the effective sysctls including the + // enable_unprivileged_ports/enable_unprivileged_icmp defaults CRI applies + // to pause, DNS, cgroup parent, labels) and have CRI send it as an + // extension of the create request; a sandboxed shim should not depend on + // the CRI layer and its PodSandboxConfig, nor re-derive CRI policy. + var pc runtime.PodSandboxConfig + if err := typeurl.UnmarshalTo(r.GetOptions(), &pc); err != nil { + return nil, fmt.Errorf("sandbox %q: options are not a CRI PodSandboxConfig: %w", id, err) + } + return configFromPodSandboxConfig(id, r.GetNetnsPath(), r.GetAnnotations(), &pc) +} + +func configFromPodSandboxConfig(id, netns string, reqAnnotations map[string]string, pc *runtime.PodSandboxConfig) (*Config, error) { + nsOpts := pc.GetLinux().GetSecurityContext().GetNamespaceOptions() + cfg := &Config{ + ID: id, + Hostname: pc.GetHostname(), + HostNetwork: nsOpts.GetNetwork() == runtime.NamespaceMode_NODE, + HostIPC: nsOpts.GetIpc() == runtime.NamespaceMode_NODE, + HostPID: nsOpts.GetPid() == runtime.NamespaceMode_NODE, + SharedPID: nsOpts.GetPid() == runtime.NamespaceMode_POD, + NetNSPath: netns, + CgroupParent: pc.GetLinux().GetCgroupParent(), + DNS: pc.GetDnsConfig(), + ShmSize: sandboxfiles.DefaultShmSize, + } + + if nsOpts.GetPid() == runtime.NamespaceMode_TARGET { + return nil, fmt.Errorf("sandbox %q: PID namespace mode TARGET is not valid for a pod sandbox: %w", id, errdefs.ErrInvalidArgument) + } + if u := nsOpts.GetUsernsOptions(); u != nil && u.GetMode() != runtime.NamespaceMode_NODE { + // The namespaces of a user namespaced pod must be owned by the pod + // user namespace, which a multithreaded shim cannot enter; CRI has + // to create them. That is a follow-up. + return nil, fmt.Errorf("sandbox %q: user namespaces are not supported yet by %s: %w", id, BinaryName, errdefs.ErrNotImplemented) + } + + if cfg.HostNetwork { + cfg.NetNSPath = "" + } else if cfg.NetNSPath == "" { + return nil, fmt.Errorf("sandbox %q: no network namespace path for a pod with its own network namespace: %w", id, errdefs.ErrInvalidArgument) + } + + if cfg.Hostname == "" { + hostname, err := os.Hostname() + if err != nil { + return nil, fmt.Errorf("sandbox %q: failed to get the node hostname: %w", id, err) + } + cfg.Hostname = hostname + } + + sysctls, err := effectiveSysctls(pc.GetLinux().GetSysctls(), cfg.HostNetwork, cfg.HostIPC) + if err != nil { + return nil, fmt.Errorf("sandbox %q: %w", id, err) + } + cfg.Sysctls = sysctls + cfg.Annotations = reqAnnotations + return cfg, nil +} + +// Sysctls are applied by the shim thread that created the pod namespaces (for +// pause, runc applied them inside the pause container), so only namespaced +// sysctls of namespaces the pod owns can be honored. + +const sysctlRoot = "/proc/sys" + +// defaultSysctls are applied to pods with their own network namespace unless +// the pod sets them, matching what containerd applies to pause sandboxes with +// its default CRI configuration (enable_unprivileged_ports and +// enable_unprivileged_icmp both on). +var defaultSysctls = map[string]string{ + "net.ipv4.ip_unprivileged_port_start": "0", + "net.ipv4.ping_group_range": "0 2147483647", +} + +// validateSysctl accepts the namespaced sysctls the shim can apply from a host +// thread that entered the pod namespaces, and rejects the rest: user.* is +// scoped to a user namespace the shim never enters, a sysctl of a namespace +// the pod shares with the host would change the host, and anything else is +// not namespaced at all. It returns the key in dotted form, the form it +// classified and the one that names the file (see sysctlPath). +func validateSysctl(key string, hostNetwork, hostIPC bool) (string, error) { + _, dotted, err := sysctlPath(key) + if err != nil { + return "", err + } + switch { + case strings.HasPrefix(dotted, "net."): + if hostNetwork { + return "", fmt.Errorf("sysctl %q is not allowed for a pod using the host network: %w", key, errdefs.ErrInvalidArgument) + } + case strings.HasPrefix(dotted, "kernel.shm"), strings.HasPrefix(dotted, "kernel.msg"), dotted == "kernel.sem", strings.HasPrefix(dotted, "fs.mqueue."): + if hostIPC { + return "", fmt.Errorf("sysctl %q is not allowed for a pod using the host IPC namespace: %w", key, errdefs.ErrInvalidArgument) + } + case dotted == "kernel.hostname", dotted == "kernel.domainname": + if hostNetwork { + return "", fmt.Errorf("sysctl %q is not allowed for a pod using the host network (and UTS) namespace: %w", key, errdefs.ErrInvalidArgument) + } + case strings.HasPrefix(dotted, "user."): + return "", fmt.Errorf("sysctl %q: user.* sysctls are not supported by %s: %w", key, BinaryName, errdefs.ErrNotImplemented) + default: + return "", fmt.Errorf("sysctl %q is not namespaced and cannot be set for a pod: %w", key, errdefs.ErrInvalidArgument) + } + return dotted, nil +} + +// effectiveSysctls validates the pod sysctls and adds the defaults. The keys +// of the result are in dotted form. +func effectiveSysctls(pod map[string]string, hostNetwork, hostIPC bool) (map[string]string, error) { + out := make(map[string]string, len(pod)+len(defaultSysctls)) + for k, v := range pod { + dotted, err := validateSysctl(k, hostNetwork, hostIPC) + if err != nil { + return nil, err + } + out[dotted] = v + } + if !hostNetwork { + for k, v := range defaultSysctls { + if _, ok := out[k]; !ok { + out[k] = v + } + } + } + return out, nil +} + +// sysctlPath maps a sysctl key to its file under /proc/sys and returns the key +// in dotted form, the form validateSysctl classifies. Keys are spelled as in +// sysctl.d(5), which is what kubelet passes through and runc accepts: the +// first separator, "." or "/", separates the components and the other +// character is literal. The rp_filter of the interface eth0.100 is therefore +// "net.ipv4.conf.eth0/100.rp_filter" or "net/ipv4/conf/eth0.100/rp_filter", +// the file /proc/sys/net/ipv4/conf/eth0.100/rp_filter. Every component must +// be a name: with "." or ".." the file would be of another class than the +// dotted form says. +func sysctlPath(key string) (path, dotted string, err error) { + swap := func(r rune) rune { + switch r { + case '.': + return '/' + case '/': + return '.' + } + return r + } + dotted = key + if i := strings.IndexAny(key, "./"); i >= 0 && key[i] == '/' { + dotted = strings.Map(swap, key) + } + rel := strings.Map(swap, dotted) + for c := range strings.SplitSeq(rel, "/") { + if c == "" || c == "." || c == ".." || strings.ContainsRune(c, 0) { + return "", "", fmt.Errorf("invalid sysctl key %q: %w", key, errdefs.ErrInvalidArgument) + } + } + return sysctlRoot + "/" + rel, dotted, nil +} + +// buildSpec synthesizes the sandbox spec CRI and NRI consume. It is not the +// spec of a process (there is none): it describes the namespaces the sandbox +// holds, from what was actually pinned, the sysctls applied to them, and the +// pod shared files as mounts. A namespace the pod shares with the host has no +// entry. Linux is always set, which is how CRI tells a sandbox without a +// process from an older Sandbox API shim that returns no spec. +// +// The PID namespace entry has no path: the sandbox holds none. With a pause +// container the pod PID namespace is the one pause is PID 1 of, and a pod +// that asks to share it (NamespaceMode_POD, the CRI default and kubelet's +// shareProcessNamespace) joins its containers to it. Holding a shared PID +// namespace needs a durable PID 1 (sandbox-init, a follow-up); until then +// the entry declares that each container gets a new PID namespace of its +// own, and CRI honors that declaration instead of failing. +func buildSpec(cfg *Config, pins Pins, mounts []specs.Mount) *specs.Spec { + s := &specs.Spec{ + Version: specs.Version, + Hostname: cfg.Hostname, + Annotations: cfg.Annotations, + Mounts: mounts, + Linux: &specs.Linux{ + Sysctl: cfg.Sysctls, + }, + } + if cfg.NetNSPath != "" { + s.Linux.Namespaces = append(s.Linux.Namespaces, specs.LinuxNamespace{Type: specs.NetworkNamespace, Path: cfg.NetNSPath}) + } + if pins.IPC != "" { + s.Linux.Namespaces = append(s.Linux.Namespaces, specs.LinuxNamespace{Type: specs.IPCNamespace, Path: pins.IPC}) + } + if pins.UTS != "" { + s.Linux.Namespaces = append(s.Linux.Namespaces, specs.LinuxNamespace{Type: specs.UTSNamespace, Path: pins.UTS}) + } + if !cfg.HostPID { + s.Linux.Namespaces = append(s.Linux.Namespaces, specs.LinuxNamespace{Type: specs.PIDNamespace}) + } + return s +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config_test.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config_test.go new file mode 100644 index 0000000000..393f5b4204 --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config_test.go @@ -0,0 +1,360 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "os" + "testing" + + api "github.com/containerd/containerd/api/runtime/sandbox/v1" + "github.com/containerd/errdefs" + "github.com/containerd/typeurl/v2" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" +) + +const ( + testID = "sandbox-id" + testNetNS = "/var/run/netns/cni-test" +) + +func testPodConfig() *runtime.PodSandboxConfig { + return &runtime.PodSandboxConfig{ + Metadata: &runtime.PodSandboxMetadata{ + Name: "pod", + Uid: "pod-uid", + Namespace: "pod-ns", + }, + Hostname: "pod-hostname", + LogDirectory: "/var/log/pods/pod", + DnsConfig: &runtime.DNSConfig{ + Servers: []string{"10.0.0.10"}, + }, + Linux: &runtime.LinuxPodSandboxConfig{ + CgroupParent: "/kubepods/pod", + SecurityContext: &runtime.LinuxSandboxSecurityContext{ + NamespaceOptions: &runtime.NamespaceOption{}, + }, + }, + } +} + +func createRequest(t *testing.T, pc *runtime.PodSandboxConfig, netns string) *api.CreateSandboxRequest { + t.Helper() + req := &api.CreateSandboxRequest{ + SandboxID: testID, + BundlePath: "/bundle", + NetnsPath: netns, + Annotations: map[string]string{"extra": "annotation"}, + } + if pc != nil { + any, err := typeurl.MarshalAnyToProto(pc) + require.NoError(t, err) + req.Options = any + } + return req +} + +func TestConfigFromRequest(t *testing.T) { + t.Run("options are required", func(t *testing.T) { + _, err := ConfigFromRequest(createRequest(t, nil, testNetNS)) + require.Error(t, err) + assert.True(t, errdefs.IsInvalidArgument(err)) + }) + + t.Run("pod namespaces", func(t *testing.T) { + cfg, err := ConfigFromRequest(createRequest(t, testPodConfig(), testNetNS)) + require.NoError(t, err) + assert.Equal(t, testID, cfg.ID) + assert.Equal(t, "pod-hostname", cfg.Hostname) + assert.False(t, cfg.HostNetwork) + assert.False(t, cfg.HostIPC) + assert.False(t, cfg.HostPID) + assert.Equal(t, testNetNS, cfg.NetNSPath) + assert.Equal(t, "/kubepods/pod", cfg.CgroupParent) + assert.Equal(t, []string{"10.0.0.10"}, cfg.DNS.GetServers()) + assert.Equal(t, defaultSysctls, cfg.Sysctls, "the pause defaults apply to a pod with its own network namespace") + assert.Equal(t, map[string]string{"extra": "annotation"}, cfg.Annotations, "the annotations of the request") + }) + + t.Run("pod network requires a network namespace path", func(t *testing.T) { + _, err := ConfigFromRequest(createRequest(t, testPodConfig(), "")) + require.Error(t, err) + assert.True(t, errdefs.IsInvalidArgument(err)) + }) + + t.Run("host network", func(t *testing.T) { + pc := testPodConfig() + pc.Linux.SecurityContext.NamespaceOptions.Network = runtime.NamespaceMode_NODE + cfg, err := ConfigFromRequest(createRequest(t, pc, "")) + require.NoError(t, err) + assert.True(t, cfg.HostNetwork) + assert.Empty(t, cfg.NetNSPath) + assert.Empty(t, cfg.Sysctls, "no network defaults for a pod in the host network namespace") + + pc.Linux.Sysctls = map[string]string{"net.ipv4.ip_forward": "1"} + _, err = ConfigFromRequest(createRequest(t, pc, "")) + require.Error(t, err, "a net sysctl would change the host") + assert.True(t, errdefs.IsInvalidArgument(err)) + }) + + t.Run("host ipc", func(t *testing.T) { + pc := testPodConfig() + pc.Linux.SecurityContext.NamespaceOptions.Ipc = runtime.NamespaceMode_NODE + cfg, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err) + assert.True(t, cfg.HostIPC) + + pc.Linux.Sysctls = map[string]string{"kernel.shm_rmid_forced": "1"} + _, err = ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.Error(t, err, "an IPC sysctl would change the host") + assert.True(t, errdefs.IsInvalidArgument(err)) + }) + + t.Run("host pid", func(t *testing.T) { + pc := testPodConfig() + pc.Linux.SecurityContext.NamespaceOptions.Pid = runtime.NamespaceMode_NODE + cfg, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err) + assert.True(t, cfg.HostPID) + }) + + t.Run("pod sysctls win over the defaults", func(t *testing.T) { + pc := testPodConfig() + pc.Linux.Sysctls = map[string]string{ + "net.ipv4.ip_unprivileged_port_start": "1024", + "kernel.shm_rmid_forced": "1", + "fs.mqueue.msg_max": "20", + } + cfg, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err) + assert.Equal(t, map[string]string{ + "net.ipv4.ip_unprivileged_port_start": "1024", + "net.ipv4.ping_group_range": "0 2147483647", + "kernel.shm_rmid_forced": "1", + "fs.mqueue.msg_max": "20", + }, cfg.Sysctls) + }) + + t.Run("slash separated sysctls", func(t *testing.T) { + // kubelet passes sysctl names with either separator through as + // written; they are classified and kept in dotted form. + pc := testPodConfig() + pc.Linux.Sysctls = map[string]string{"net/ipv4/ping_group_range": "1 1", "kernel/shm_rmid_forced": "1"} + cfg, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err) + assert.Equal(t, map[string]string{ + "net.ipv4.ip_unprivileged_port_start": "0", + "net.ipv4.ping_group_range": "1 1", + "kernel.shm_rmid_forced": "1", + }, cfg.Sysctls) + + pc.Linux.SecurityContext.NamespaceOptions.Ipc = runtime.NamespaceMode_NODE + _, err = ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.Error(t, err, "an IPC sysctl would change the host, however it is spelled") + assert.True(t, errdefs.IsInvalidArgument(err)) + }) + + t.Run("shared pid namespace is accepted but not shared", func(t *testing.T) { + // NamespaceMode_POD is the zero value, so it is what a pod config + // without explicit namespace options (crictl, critest) asks for. + pc := testPodConfig() + pc.Linux.SecurityContext.NamespaceOptions.Pid = runtime.NamespaceMode_POD + cfg, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err) + assert.True(t, cfg.SharedPID) + + pc.Linux.SecurityContext.NamespaceOptions.Pid = runtime.NamespaceMode_CONTAINER + cfg, err = ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err) + assert.False(t, cfg.SharedPID) + }) + + t.Run("target pid namespace is invalid for a sandbox", func(t *testing.T) { + pc := testPodConfig() + pc.Linux.SecurityContext.NamespaceOptions.Pid = runtime.NamespaceMode_TARGET + _, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.Error(t, err) + assert.True(t, errdefs.IsInvalidArgument(err)) + }) + + t.Run("user namespaces are not supported yet", func(t *testing.T) { + pc := testPodConfig() + pc.Linux.SecurityContext.NamespaceOptions.UsernsOptions = &runtime.UserNamespace{Mode: runtime.NamespaceMode_POD} + _, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.Error(t, err) + assert.True(t, errdefs.IsNotImplemented(err)) + + pc.Linux.SecurityContext.NamespaceOptions.UsernsOptions = &runtime.UserNamespace{Mode: runtime.NamespaceMode_NODE} + _, err = ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err, "the host user namespace is the default") + }) + + t.Run("unsupported sysctls", func(t *testing.T) { + pc := testPodConfig() + pc.Linux.Sysctls = map[string]string{"user.max_user_namespaces": "10"} + _, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.Error(t, err) + assert.True(t, errdefs.IsNotImplemented(err), "user.* needs the pod user namespace") + + pc.Linux.Sysctls = map[string]string{"vm.swappiness": "10"} + _, err = ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.Error(t, err) + assert.True(t, errdefs.IsInvalidArgument(err), "not namespaced") + + pc.Linux.Sysctls = map[string]string{".": "x"} + _, err = ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.Error(t, err) + assert.True(t, errdefs.IsInvalidArgument(err), "not a key") + }) + + t.Run("hostname defaults to the node hostname", func(t *testing.T) { + pc := testPodConfig() + pc.Hostname = "" + cfg, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err) + hostname, err := os.Hostname() + require.NoError(t, err) + assert.Equal(t, hostname, cfg.Hostname) + }) +} + +func TestSysctlPath(t *testing.T) { + // Either separator, as in sysctl.d(5): the first one separates the + // components and the other character is literal, which is how a dotted + // interface name is spelled. The dotted form is what gets classified. + for key, want := range map[string][2]string{ + "net.ipv4.ip_unprivileged_port_start": {"/proc/sys/net/ipv4/ip_unprivileged_port_start", "net.ipv4.ip_unprivileged_port_start"}, + "net/ipv4/ip_unprivileged_port_start": {"/proc/sys/net/ipv4/ip_unprivileged_port_start", "net.ipv4.ip_unprivileged_port_start"}, + "net.ipv4.conf.eth0/100.rp_filter": {"/proc/sys/net/ipv4/conf/eth0.100/rp_filter", "net.ipv4.conf.eth0/100.rp_filter"}, + "net/ipv4/conf/eth0.100/rp_filter": {"/proc/sys/net/ipv4/conf/eth0.100/rp_filter", "net.ipv4.conf.eth0/100.rp_filter"}, + "kernel/shm_rmid_forced": {"/proc/sys/kernel/shm_rmid_forced", "kernel.shm_rmid_forced"}, + } { + p, dotted, err := sysctlPath(key) + require.NoError(t, err, "key %q", key) + assert.Equal(t, want[0], p, "key %q", key) + assert.Equal(t, want[1], dotted, "key %q", key) + } + + // No component may be empty, "." or "..": the file would not be under + // the directory the dotted form names. "net.//.kernel.shm_rmid_forced" + // (net/../kernel/shm_rmid_forced with slashes) is an IPC sysctl spelled + // as a net one. A NUL byte cannot be part of a path either. + for _, key := range []string{"", ".", "..", "/", "net.", ".net.ipv4.ip_forward", "net..ipv4.ip_forward", "net.//.kernel.shm_rmid_forced", "net/../kernel/shm_rmid_forced", "net./../kernel.shm_rmid_forced", "net.\x00"} { + _, _, err := sysctlPath(key) + assert.ErrorIs(t, err, errdefs.ErrInvalidArgument, "key %q", key) + } +} + +func TestValidateSysctl(t *testing.T) { + for _, tc := range []struct { + key string + hostNetwork, hostIPC bool + dotted string + }{ + {key: "net.ipv4.ip_forward", dotted: "net.ipv4.ip_forward"}, + {key: "kernel.shm_rmid_forced", dotted: "kernel.shm_rmid_forced"}, + {key: "kernel.sem", dotted: "kernel.sem"}, + {key: "fs.mqueue.msg_max", dotted: "fs.mqueue.msg_max"}, + {key: "kernel.hostname", dotted: "kernel.hostname"}, + {key: "kernel/shm_rmid_forced", dotted: "kernel.shm_rmid_forced"}, + {key: "net/ipv4/conf/eth0.100/rp_filter", dotted: "net.ipv4.conf.eth0/100.rp_filter"}, + // IPC sysctls are fine with the host network, net sysctls with the + // host IPC namespace. + {key: "kernel.shm_rmid_forced", hostNetwork: true, dotted: "kernel.shm_rmid_forced"}, + {key: "net.ipv4.ip_forward", hostIPC: true, dotted: "net.ipv4.ip_forward"}, + } { + dotted, err := validateSysctl(tc.key, tc.hostNetwork, tc.hostIPC) + require.NoError(t, err, "key %q", tc.key) + assert.Equal(t, tc.dotted, dotted, "key %q", tc.key) + } + + for _, tc := range []struct { + key string + hostNetwork, hostIPC bool + wantErr error + }{ + {key: "net.ipv4.ip_forward", hostNetwork: true, wantErr: errdefs.ErrInvalidArgument}, + {key: "net/ipv4/ip_forward", hostNetwork: true, wantErr: errdefs.ErrInvalidArgument}, + // The UTS namespace is shared with the host along with the network one. + {key: "kernel.hostname", hostNetwork: true, wantErr: errdefs.ErrInvalidArgument}, + {key: "kernel.domainname", hostNetwork: true, wantErr: errdefs.ErrInvalidArgument}, + {key: "kernel.shm_rmid_forced", hostIPC: true, wantErr: errdefs.ErrInvalidArgument}, + {key: "kernel/shm_rmid_forced", hostIPC: true, wantErr: errdefs.ErrInvalidArgument}, + {key: "kernel.msgmax", hostIPC: true, wantErr: errdefs.ErrInvalidArgument}, + {key: "fs.mqueue.msg_max", hostIPC: true, wantErr: errdefs.ErrInvalidArgument}, + // Like runc, only kernel.sem itself counts as an IPC sysctl. + {key: "kernel.sem_next_id", wantErr: errdefs.ErrInvalidArgument}, + {key: "user.max_user_namespaces", wantErr: errdefs.ErrNotImplemented}, + {key: "vm.swappiness", wantErr: errdefs.ErrInvalidArgument}, + {key: "net.\x00", wantErr: errdefs.ErrInvalidArgument}, + } { + _, err := validateSysctl(tc.key, tc.hostNetwork, tc.hostIPC) + assert.ErrorIs(t, err, tc.wantErr, "key %q, host network %v, host IPC %v", tc.key, tc.hostNetwork, tc.hostIPC) + } +} + +func TestBuildSpec(t *testing.T) { + pins := Pins{IPC: "/bundle/ns/ipc", UTS: "/bundle/ns/uts"} + mounts := []specs.Mount{{Destination: "/etc/hosts", Type: "bind", Source: "/bundle/hosts", Options: []string{"rbind"}}} + sysctls := map[string]string{"net.ipv4.ip_forward": "1"} + annots := map[string]string{"io.kubernetes.cri.sandbox-id": testID} + + t.Run("pod namespaces", func(t *testing.T) { + cfg := &Config{ID: testID, Hostname: "pod", NetNSPath: testNetNS, Sysctls: sysctls, Annotations: annots} + spec := buildSpec(cfg, pins, mounts) + assert.Equal(t, specs.Version, spec.Version) + assert.Equal(t, "pod", spec.Hostname) + assert.Equal(t, annots, spec.Annotations) + assert.Equal(t, mounts, spec.Mounts) + require.NotNil(t, spec.Linux) + assert.Equal(t, sysctls, spec.Linux.Sysctl) + assert.Equal(t, []specs.LinuxNamespace{ + {Type: specs.NetworkNamespace, Path: testNetNS}, + {Type: specs.IPCNamespace, Path: pins.IPC}, + {Type: specs.UTSNamespace, Path: pins.UTS}, + {Type: specs.PIDNamespace}, + }, spec.Linux.Namespaces) + }) + + t.Run("host network keeps only the IPC namespace", func(t *testing.T) { + cfg := &Config{ID: testID, Hostname: "node", HostNetwork: true} + spec := buildSpec(cfg, Pins{IPC: pins.IPC}, nil) + assert.Equal(t, []specs.LinuxNamespace{{Type: specs.IPCNamespace, Path: pins.IPC}, {Type: specs.PIDNamespace}}, spec.Linux.Namespaces) + }) + + t.Run("host ipc keeps the network and UTS namespaces", func(t *testing.T) { + cfg := &Config{ID: testID, Hostname: "pod", HostIPC: true, NetNSPath: testNetNS} + spec := buildSpec(cfg, Pins{UTS: pins.UTS}, nil) + assert.Equal(t, []specs.LinuxNamespace{ + {Type: specs.NetworkNamespace, Path: testNetNS}, + {Type: specs.UTSNamespace, Path: pins.UTS}, + {Type: specs.PIDNamespace}, + }, spec.Linux.Namespaces) + }) + + t.Run("all host namespaces still carry a Linux section", func(t *testing.T) { + cfg := &Config{ID: testID, Hostname: "node", HostNetwork: true, HostIPC: true, HostPID: true} + spec := buildSpec(cfg, Pins{}, nil) + require.NotNil(t, spec.Linux, "CRI tells a processless sandbox by the Linux section") + assert.Empty(t, spec.Linux.Namespaces, "no PID namespace declaration for a host PID pod") + }) +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/doc.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/doc.go new file mode 100644 index 0000000000..780d9900d9 --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/doc.go @@ -0,0 +1,100 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package sandbox implements the Sandbox API (runtime.sandbox.v1) for a pod +// that has no pause container. +// +// # Configuration +// +// The shim serves the io.containerd.runc.v2 runtime type from a different +// binary, so CRI treats the handler exactly as runc (runc options, features, +// cgroup driver); the handler names the binary and the sandboxer: +// +// [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] +// runtime_type = "io.containerd.runc.v2" +// runtime_path = "/usr/local/bin/containerd-shim-sandboxed-runc-v2" +// sandboxer = "shim" +// disable_pause_image_pull = true +// +// # What a pause sandbox does today +// +// With the podsandbox controller a pod sandbox is a container: CRI pulls the +// pause image, builds an OCI spec for it (hostname, namespaces, sysctls, +// cgroup, security options), creates the pod shared files under the CRI state +// directory (/sandboxes//{hostname,hosts,resolv.conf,shm}) and runs +// the pause process through the runc shim. The pause process exists only to +// keep the pod network, IPC and UTS namespaces alive: containers of the pod +// join /proc//ns/{net,ipc,uts} and CRI bind-mounts the shared +// files, which it created itself, into every container. +// +// # What this package does instead +// +// The shim is the sandbox. CreateSandbox receives the CRI PodSandboxConfig +// (as the sandbox options) and the network namespace CRI created and +// configured with CNI, and sets the pod up without a process: +// +// - Namespaces (namespaces.go). A locked OS thread enters the CRI network +// namespace, unshares an IPC and a UTS namespace, sets the pod hostname, +// writes the pod sysctls (net.* land in the pod network namespace, +// kernel.shm*, kernel.msg*, kernel.sem and fs.mqueue.* in the pod IPC +// namespace) and bind-mounts its /proc/self/task//ns/{ipc,uts} onto +// files under /ns. The thread is then discarded; the bind mounts, +// the pins, keep the namespaces alive the same way CRI keeps the network +// namespace alive under /var/run/netns. A namespace the pod shares with +// the host (hostNetwork, hostIPC) is neither created nor pinned. +// +// - Shared files (bundle.go). The shim writes hostname, copies the host +// hosts file, writes resolv.conf from the pod DNS config (or copies the +// host one) and mounts the pod shm tmpfs, all under its bundle instead of +// the CRI state directory, and lists them as Mounts of the sandbox spec. +// +// - Spec (buildSpec in config.go). StartSandbox returns pid 0 and a +// synthesized OCI spec that carries the namespace paths, the sysctls, the +// shared file mounts and the annotations of the request. It describes what +// the sandbox holds; there is no process it could describe. +// +// CRI consumes that spec where it used the pause pid before: a sandbox +// reporting pid 0 with a Linux section in its spec joins containers to the +// namespace paths of the spec (a namespace missing from the spec is the host +// one when the pod asked for it, and an error otherwise), and bind-mounts the +// shared files from the spec mount sources when nothing exists at the fixed +// CRI paths. Sandboxes with a pid, and older Sandbox API shims returning no +// spec, keep the pid derived paths unchanged. +// +// # Lifetime and cleanup +// +// One shim process serves one pod: its sandbox and, through the wrapped task +// service of containerd-shim-runc-v2 (task.go in package main), the runc tasks +// of its containers. StopSandbox releases what the sandbox holds and +// ShutdownSandbox ends the shim; CRI stops and removes the containers of a +// pod before either. Everything the shim creates lives under its bundle at +// fixed paths, so Cleanup needs no state and runs the same way on create +// failure, on stop and from "shim delete". +// +// # Not supported yet +// +// A shared pod PID namespace needs a durable PID 1 (sandbox-init); pods +// asking for one get per-container PID namespaces, with a warning (see +// buildSpec). User namespaced pods need their namespaces created inside the +// pod user namespace, which a multithreaded shim cannot enter; they are +// rejected, as are user.* sysctls. The shim requires cgroup v2 and does not +// run rootless. SELinux labels are not applied yet: the pod label allocation +// moves into the CRI layer with +// https://github.com/containerd/containerd/pull/14108, after which the shim +// applies the labels it is handed (see the TODO in CreateSandbox). +package sandbox diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/namespaces.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/namespaces.go new file mode 100644 index 0000000000..4fb8936075 --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/namespaces.go @@ -0,0 +1,233 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "errors" + "fmt" + "io/fs" + "maps" + "os" + "path/filepath" + "runtime" + "slices" + + "golang.org/x/sys/unix" +) + +// Pod namespaces without a process; see the package doc. The approach follows +// nerdbox's shared resources implementation (github.com/containerd/nerdbox, +// internal/vminit/sharedresources). + +const ( + // pinDir is the bundle directory holding the namespace pins. + pinDir = "ns" + ipcPin = "ipc" + utsPin = "uts" +) + +// Pins are the bind-mounted namespace files a sandbox holds. An empty path +// means the pod uses the host namespace of that type. +type Pins struct { + IPC string + UTS string +} + +// setupNamespaces creates and pins the pod namespaces under the bundle and +// applies the hostname and sysctls to them. On error the caller runs Cleanup. +func setupNamespaces(bundle string, cfg *Config) (Pins, error) { + ipc, uts := !cfg.HostIPC, !cfg.HostNetwork + if !ipc && !uts { + // Nothing to hold, and the sysctl policy leaves nothing to apply to + // namespaces the pod shares with the host. + return Pins{}, nil + } + dir := filepath.Join(bundle, pinDir) + if err := os.Mkdir(dir, 0o700); err != nil && !errors.Is(err, fs.ErrExist) { + return Pins{}, fmt.Errorf("failed to create namespace pin directory: %w", err) + } + + var pins Pins + errCh := make(chan error, 1) + go func() { + // The thread is locked while it is in the pod namespaces and put + // back into the host namespaces before it is released, whether the + // work succeeded or not: a thread Go cannot discard (the main thread) + // must never stay in the pod. The pins keep the namespaces alive. + runtime.LockOSThread() + host, err := openThreadNamespaces() + if err != nil { + errCh <- err + return + } + err = func() error { + if cfg.NetNSPath != "" { + // Entered first so that net sysctls apply to the pod network + // namespace. + if err := setns(cfg.NetNSPath, unix.CLONE_NEWNET); err != nil { + return err + } + } + var flags int + if ipc { + flags |= unix.CLONE_NEWIPC + } + if uts { + flags |= unix.CLONE_NEWUTS + } + if err := unix.Unshare(flags); err != nil { + return fmt.Errorf("failed to create pod namespaces: %w", err) + } + if uts { + if err := unix.Sethostname([]byte(cfg.Hostname)); err != nil { + return fmt.Errorf("failed to set hostname %q: %w", cfg.Hostname, err) + } + } + for _, k := range slices.Sorted(maps.Keys(cfg.Sysctls)) { + if err := writeSysctl(k, cfg.Sysctls[k]); err != nil { + return err + } + } + tid := unix.Gettid() + if ipc { + p := filepath.Join(dir, ipcPin) + if err := pin(threadNamespacePath(tid, "ipc"), p); err != nil { + return err + } + pins.IPC = p + } + if uts { + p := filepath.Join(dir, utsPin) + if err := pin(threadNamespacePath(tid, "uts"), p); err != nil { + return err + } + pins.UTS = p + } + return nil + }() + if rerr := host.restore(); rerr != nil { + // The thread stays locked and is discarded with the goroutine. + errCh <- errors.Join(err, rerr) + return + } + runtime.UnlockOSThread() + errCh <- err + }() + if err := <-errCh; err != nil { + return Pins{}, err + } + return pins, nil +} + +// threadNamespaces holds the namespace files of a thread before it moves, +// so that it can be moved back. +type threadNamespaces struct { + net, ipc, uts int +} + +func openThreadNamespaces() (*threadNamespaces, error) { + t := &threadNamespaces{net: -1, ipc: -1, uts: -1} + for _, ns := range []struct { + name string + fd *int + }{{"net", &t.net}, {"ipc", &t.ipc}, {"uts", &t.uts}} { + fd, err := unix.Open("/proc/thread-self/ns/"+ns.name, unix.O_RDONLY|unix.O_CLOEXEC, 0) + if err != nil { + t.close() + return nil, fmt.Errorf("failed to open the %s namespace of the thread: %w", ns.name, err) + } + *ns.fd = fd + } + return t, nil +} + +// restore moves the thread back into its original namespaces and closes them. +func (t *threadNamespaces) restore() error { + defer t.close() + var errs []error + for _, ns := range []struct { + fd int + flag int + which string + }{{t.uts, unix.CLONE_NEWUTS, "uts"}, {t.ipc, unix.CLONE_NEWIPC, "ipc"}, {t.net, unix.CLONE_NEWNET, "net"}} { + if err := unix.Setns(ns.fd, ns.flag); err != nil { + errs = append(errs, fmt.Errorf("failed to move the thread back into the host %s namespace: %w", ns.which, err)) + } + } + return errors.Join(errs...) +} + +func (t *threadNamespaces) close() { + for _, fd := range []int{t.net, t.ipc, t.uts} { + if fd >= 0 { + unix.Close(fd) + } + } + t.net, t.ipc, t.uts = -1, -1, -1 +} + +// setns moves the calling thread into the namespace pinned at path. +func setns(path string, nstype int) error { + fd, err := unix.Open(path, unix.O_RDONLY|unix.O_CLOEXEC, 0) + if err != nil { + return fmt.Errorf("failed to open namespace %s: %w", path, err) + } + defer unix.Close(fd) + if err := unix.Setns(fd, nstype); err != nil { + return fmt.Errorf("failed to enter namespace %s: %w", path, err) + } + return nil +} + +// threadNamespacePath is the namespace file of one thread. /proc/self/ns +// would name the namespaces of the main thread, not of the thread that +// changed them. +func threadNamespacePath(tid int, nsType string) string { + return fmt.Sprintf("/proc/%d/task/%d/ns/%s", os.Getpid(), tid, nsType) +} + +// pin bind-mounts the namespace file src onto a new empty file at target so +// that the namespace persists without a process in it. +func pin(src, target string) error { + f, err := os.OpenFile(target, os.O_RDONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return fmt.Errorf("failed to create namespace pin %s: %w", target, err) + } + f.Close() + if err := unix.Mount(src, target, "none", unix.MS_BIND, ""); err != nil { + os.Remove(target) + return fmt.Errorf("failed to pin namespace %s at %s: %w", src, target, err) + } + return nil +} + +// writeSysctl sets a sysctl for the namespaces of the calling thread: the +// files under /proc/sys/net resolve against the network namespace of the +// caller and the IPC ones against its IPC namespace, so this must run on the +// thread that entered the pod namespaces. +func writeSysctl(key, value string) error { + p, _, err := sysctlPath(key) + if err != nil { + return err + } + if err := os.WriteFile(p, []byte(value), 0o644); err != nil { + return fmt.Errorf("failed to set sysctl %s=%q: %w", key, value, err) + } + return nil +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/namespaces_test.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/namespaces_test.go new file mode 100644 index 0000000000..4f78ff665a --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/namespaces_test.go @@ -0,0 +1,152 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/moby/sys/mountinfo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" + + "github.com/containerd/containerd/v2/pkg/testutil" +) + +// inNamespace runs fn on a thread that entered the namespace pinned at pin +// and returns what it reads. The thread is locked and discarded afterwards, +// like the one that created the namespace. +func inNamespace(t *testing.T, pin string, nsType int, fn func() (string, error)) string { + t.Helper() + var ( + result string + err error + ) + done := make(chan struct{}) + go func() { + defer close(done) + runtime.LockOSThread() + var fd int + if fd, err = unix.Open(pin, unix.O_RDONLY|unix.O_CLOEXEC, 0); err != nil { + return + } + defer unix.Close(fd) + if err = unix.Setns(fd, nsType); err != nil { + return + } + result, err = fn() + }() + <-done + require.NoError(t, err, "in namespace %s", pin) + return result +} + +func inode(t *testing.T, path string) uint64 { + t.Helper() + var st unix.Stat_t + require.NoError(t, unix.Stat(path, &st)) + return st.Ino +} + +func assertNoMountsUnder(t *testing.T, dir string) { + t.Helper() + mounts, err := mountinfo.GetMounts(mountinfo.PrefixFilter(dir)) + require.NoError(t, err) + assert.Empty(t, mounts, "no mounts left under %s", dir) +} + +func TestSetupNamespaces(t *testing.T) { + testutil.RequiresRoot(t) + ctx := context.Background() + bundle := t.TempDir() + t.Cleanup(func() { _ = Cleanup(ctx, bundle) }) + + pins, err := setupNamespaces(bundle, &Config{ + Hostname: "sandbox-test", + Sysctls: map[string]string{"kernel.shm_rmid_forced": "1"}, + }) + require.NoError(t, err) + assert.Equal(t, filepath.Join(bundle, pinDir, ipcPin), pins.IPC) + assert.Equal(t, filepath.Join(bundle, pinDir, utsPin), pins.UTS) + + // The pins are namespace files distinct from the host namespaces. + for _, tc := range []struct{ pin, nsType string }{{pins.IPC, "ipc"}, {pins.UTS, "uts"}} { + var st unix.Statfs_t + require.NoError(t, unix.Statfs(tc.pin, &st)) + assert.EqualValues(t, unix.NSFS_MAGIC, st.Type, "%s pin is an nsfs mount", tc.nsType) + assert.NotEqual(t, inode(t, "/proc/self/ns/"+tc.nsType), inode(t, tc.pin), "%s namespace is not the host one", tc.nsType) + } + + // The pod UTS namespace carries the pod hostname; the host keeps its own. + hostHostname, err := os.Hostname() + require.NoError(t, err) + podHostname := inNamespace(t, pins.UTS, unix.CLONE_NEWUTS, func() (string, error) { + var u unix.Utsname + if err := unix.Uname(&u); err != nil { + return "", err + } + return unix.ByteSliceToString(u.Nodename[:]), nil + }) + assert.Equal(t, "sandbox-test", podHostname) + current, err := os.Hostname() + require.NoError(t, err) + assert.Equal(t, hostHostname, current) + + // The IPC sysctl was applied inside the pod IPC namespace. + value := inNamespace(t, pins.IPC, unix.CLONE_NEWIPC, func() (string, error) { + b, err := os.ReadFile("/proc/sys/kernel/shm_rmid_forced") + return strings.TrimSpace(string(b)), err + }) + assert.Equal(t, "1", value) + + // Cleanup detaches the pins and removes them; a second run is a no-op. + require.NoError(t, Cleanup(ctx, bundle)) + for _, p := range []string{pins.IPC, pins.UTS, filepath.Join(bundle, pinDir)} { + _, err := os.Stat(p) + assert.True(t, os.IsNotExist(err), "%s should be gone", p) + } + assertNoMountsUnder(t, bundle) + require.NoError(t, Cleanup(ctx, bundle)) +} + +func TestSetupNamespacesHostModes(t *testing.T) { + testutil.RequiresRoot(t) + bundle := t.TempDir() + + // A pod in the host network and IPC namespaces holds nothing. + pins, err := setupNamespaces(bundle, &Config{HostNetwork: true, HostIPC: true}) + require.NoError(t, err) + assert.Equal(t, Pins{}, pins) + _, err = os.Stat(filepath.Join(bundle, pinDir)) + assert.True(t, os.IsNotExist(err), "no pin directory without pins") + + // Host network with a pod IPC namespace: only the IPC pin exists. + pins, err = setupNamespaces(bundle, &Config{HostNetwork: true}) + require.NoError(t, err) + t.Cleanup(func() { _ = Cleanup(context.Background(), bundle) }) + assert.NotEmpty(t, pins.IPC) + assert.Empty(t, pins.UTS) + _, err = os.Stat(filepath.Join(bundle, pinDir, utsPin)) + assert.True(t, os.IsNotExist(err)) +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service.go new file mode 100644 index 0000000000..e08b3afeae --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service.go @@ -0,0 +1,366 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "fmt" + "sync" + "time" + + cgroupsv2 "github.com/containerd/cgroups/v3/cgroup2" + api "github.com/containerd/containerd/api/runtime/sandbox/v1" + "github.com/containerd/containerd/api/types" + "github.com/containerd/errdefs" + "github.com/containerd/errdefs/pkg/errgrpc" + "github.com/containerd/log" + "github.com/containerd/platforms" + "github.com/containerd/plugin" + "github.com/containerd/plugin/registry" + "github.com/containerd/ttrpc" + "github.com/containerd/typeurl/v2" + "github.com/moby/sys/userns" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "google.golang.org/protobuf/types/known/anypb" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/containerd/containerd/v2/pkg/protobuf" + "github.com/containerd/containerd/v2/pkg/shutdown" + "github.com/containerd/containerd/v2/plugins" +) + +// BinaryName is the name of this shim, used in messages. It serves the +// io.containerd.runc.v2 runtime type; a runtime handler reaches it through +// runtime_path. +const BinaryName = "containerd-shim-sandboxed-runc-v2" + +func init() { + registry.Register(&plugin.Registration{ + Type: plugins.TTRPCPlugin, + ID: "sandbox", + Requires: []plugin.Type{ + plugins.InternalPlugin, + }, + InitFn: func(ic *plugin.InitContext) (any, error) { + ss, err := ic.GetByID(plugins.InternalPlugin, "shutdown") + if err != nil { + return nil, err + } + return NewService(ss.(shutdown.Service)), nil + }, + }) +} + +// Service implements the sandbox ttrpc service. containerd starts one shim per +// sandbox, so a Service hosts at most one. +type Service struct { + shutdown shutdown.Service + + mu sync.Mutex + sandbox *instance +} + +type instance struct { + cfg *Config + bundle string + pins Pins + spec *anypb.Any + createdAt time.Time + + started bool + stopped bool + exitedAt time.Time + // done is closed once the sandbox is stopped. + done chan struct{} +} + +var _ api.TTRPCSandboxService = (*Service)(nil) + +// NewService returns a sandbox service. sd ends the shim on ShutdownSandbox. +func NewService(sd shutdown.Service) *Service { + return &Service{shutdown: sd} +} + +// RegisterTTRPC implements shim.TTRPCService. +func (s *Service) RegisterTTRPC(server *ttrpc.Server) error { + api.RegisterTTRPCSandboxService(server, s) + return nil +} + +// findSandbox returns the hosted sandbox if it is the one asked for. The +// caller holds s.mu. +func (s *Service) findSandbox(id string) (*instance, error) { + if s.sandbox == nil || s.sandbox.cfg.ID != id { + return nil, fmt.Errorf("sandbox %q: %w", id, errdefs.ErrNotFound) + } + return s.sandbox, nil +} + +// CreateSandbox sets the pod up: namespaces, hostname, sysctls and shared +// files. Nothing is left behind on failure. +func (s *Service) CreateSandbox(ctx context.Context, r *api.CreateSandboxRequest) (_ *api.CreateSandboxResponse, retErr error) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.sandbox != nil { + return nil, errgrpc.ToGRPCf(errdefs.ErrAlreadyExists, "shim already hosts sandbox %q", s.sandbox.cfg.ID) + } + if userns.RunningInUserNS() { + return nil, errgrpc.ToGRPCf(errdefs.ErrNotImplemented, "%s does not support rootless containerd", BinaryName) + } + bundle := r.GetBundlePath() + if bundle == "" { + return nil, errgrpc.ToGRPCf(errdefs.ErrInvalidArgument, "sandbox %q: no bundle path", r.GetSandboxID()) + } + cfg, err := ConfigFromRequest(r) + if err != nil { + return nil, errgrpc.ToGRPC(err) + } + log.G(ctx).WithField("id", cfg.ID).Warnf("%s is experimental: the pod sandbox runs without a pause container", BinaryName) + if cfg.SharedPID { + log.G(ctx).WithField("id", cfg.ID).Warnf("%s cannot share a pod PID namespace yet: the containers of this pod get their own PID namespaces", BinaryName) + } + + defer func() { + if retErr != nil { + if err := Cleanup(ctx, bundle); err != nil { + log.G(ctx).WithError(err).WithField("id", cfg.ID).Warn("failed to clean up after a failed sandbox creation") + } + } + }() + + pins, err := setupNamespaces(bundle, cfg) + if err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to set up the pod namespaces") + } + // TODO: SELinux. The shim allocates no pod process label and CRI does not + // allocate one for a Sandbox API shim yet, so on an enforcing host the + // containers of the pod without explicit SELinux options get independent + // MCS labels and relabel the shared files and /dev/shm in turn, and the + // pod-level SELinux options are not applied. Waits for + // https://github.com/containerd/containerd/pull/14108, which allocates the + // pod label in the CRI layer; the shim then applies the labels it is + // handed to the files it creates. + mounts, err := setupFiles(bundle, cfg) + if err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to set up the pod files") + } + spec, err := typeurl.MarshalAnyToProto(buildSpec(cfg, pins, mounts)) + if err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to marshal the sandbox spec") + } + + s.sandbox = &instance{ + cfg: cfg, + bundle: bundle, + pins: pins, + spec: spec, + createdAt: time.Now(), + done: make(chan struct{}), + } + return &api.CreateSandboxResponse{}, nil +} + +// StartSandbox marks the sandbox running. There is no process to start: the +// pid is 0 and the spec describes what the sandbox holds. Starting it again +// returns the same response. +func (s *Service) StartSandbox(ctx context.Context, r *api.StartSandboxRequest) (*api.StartSandboxResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + inst, err := s.findSandbox(r.GetSandboxID()) + if err != nil { + return nil, errgrpc.ToGRPC(err) + } + if inst.stopped { + return nil, errgrpc.ToGRPCf(errdefs.ErrFailedPrecondition, "sandbox %q is stopped", inst.cfg.ID) + } + inst.started = true + return &api.StartSandboxResponse{ + Pid: 0, + CreatedAt: protobuf.ToTimestamp(inst.createdAt), + Spec: inst.spec, + }, nil +} + +// Platform implements the sandbox service: the sandbox runs on the host. +func (s *Service) Platform(ctx context.Context, r *api.PlatformRequest) (*api.PlatformResponse, error) { + return &api.PlatformResponse{ + Platform: types.OCIPlatformToProto([]ocispec.Platform{platforms.DefaultSpec()})[0], + }, nil +} + +// StopSandbox releases the pod namespaces and files. Containers still running +// keep the namespaces they joined until they exit; CRI stops them first. +// Stopping twice is fine. +func (s *Service) StopSandbox(ctx context.Context, r *api.StopSandboxRequest) (*api.StopSandboxResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + inst, err := s.findSandbox(r.GetSandboxID()) + if err != nil { + return nil, errgrpc.ToGRPC(err) + } + if err := s.stop(ctx, inst); err != nil { + return nil, errgrpc.ToGRPC(err) + } + return &api.StopSandboxResponse{}, nil +} + +// stop releases what the sandbox holds and marks it stopped. The caller holds +// s.mu. +func (s *Service) stop(ctx context.Context, inst *instance) error { + if inst.stopped { + return nil + } + if err := Cleanup(ctx, inst.bundle); err != nil { + return fmt.Errorf("failed to clean up sandbox %q: %w", inst.cfg.ID, err) + } + inst.stopped = true + inst.exitedAt = time.Now() + close(inst.done) + return nil +} + +// WaitSandbox blocks until the sandbox is stopped. +func (s *Service) WaitSandbox(ctx context.Context, r *api.WaitSandboxRequest) (*api.WaitSandboxResponse, error) { + s.mu.Lock() + inst, err := s.findSandbox(r.GetSandboxID()) + s.mu.Unlock() + if err != nil { + return nil, errgrpc.ToGRPC(err) + } + select { + case <-inst.done: + case <-ctx.Done(): + return nil, errgrpc.ToGRPC(ctx.Err()) + } + return &api.WaitSandboxResponse{ + ExitStatus: 0, + ExitedAt: protobuf.ToTimestamp(inst.exitedAt), + }, nil +} + +// SandboxStatus reports the lifecycle state: ready from StartSandbox until the +// sandbox is stopped, not ready before and after. The verbose info carries +// only shim specific keys; the CRI server builds its own verbose document. +func (s *Service) SandboxStatus(ctx context.Context, r *api.SandboxStatusRequest) (*api.SandboxStatusResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + inst, err := s.findSandbox(r.GetSandboxID()) + if err != nil { + return nil, errgrpc.ToGRPC(err) + } + resp := &api.SandboxStatusResponse{ + SandboxID: inst.cfg.ID, + Pid: 0, + State: runtime.PodSandboxState_SANDBOX_NOTREADY.String(), + CreatedAt: protobuf.ToTimestamp(inst.createdAt), + } + switch { + case inst.stopped: + resp.ExitedAt = protobuf.ToTimestamp(inst.exitedAt) + case inst.started: + resp.State = runtime.PodSandboxState_SANDBOX_READY.String() + } + if r.GetVerbose() { + resp.Info = make(map[string]string, 4) + for _, kv := range [][2]string{ + {BinaryName + "/netns", inst.cfg.NetNSPath}, + {BinaryName + "/ipcns", inst.pins.IPC}, + {BinaryName + "/utsns", inst.pins.UTS}, + {BinaryName + "/hostname", inst.cfg.Hostname}, + } { + if kv[1] != "" { + resp.Info[kv[0]] = kv[1] + } + } + } + return resp, nil +} + +// PingSandbox implements the sandbox service. +func (s *Service) PingSandbox(ctx context.Context, r *api.PingRequest) (*api.PingResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + if _, err := s.findSandbox(r.GetSandboxID()); err != nil { + return nil, errgrpc.ToGRPC(err) + } + return &api.PingResponse{}, nil +} + +// ShutdownSandbox stops the sandbox if needed, forgets it and ends the shim. +// The containers of the pod are gone by then: CRI removes them before it +// removes the pod. Shutting down twice, or a shim that hosts no sandbox (its +// creation failed), is fine; a request for another sandbox than the hosted +// one is NotFound and leaves it running. +func (s *Service) ShutdownSandbox(ctx context.Context, r *api.ShutdownSandboxRequest) (*api.ShutdownSandboxResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.sandbox != nil { + inst, err := s.findSandbox(r.GetSandboxID()) + if err != nil { + return nil, errgrpc.ToGRPC(err) + } + if err := s.stop(ctx, inst); err != nil { + return nil, errgrpc.ToGRPC(err) + } + s.sandbox = nil + } + s.shutdown.Shutdown() + return &api.ShutdownSandboxResponse{}, nil +} + +// SandboxMetrics reports the cgroup v2 stats of the pod cgroup, encoded the +// way CRI expects sandbox metrics (a *cgroup2/stats.Metrics). The sandbox has +// no process of its own, so the aggregate pod cgroup is the only meaningful +// source. CRI polls this every second; the cgroup files are read outside the +// lock. +func (s *Service) SandboxMetrics(ctx context.Context, r *api.SandboxMetricsRequest) (*api.SandboxMetricsResponse, error) { + s.mu.Lock() + inst, err := s.findSandbox(r.GetSandboxID()) + s.mu.Unlock() + if err != nil { + return nil, errgrpc.ToGRPC(err) + } + id, cgroupParent := inst.cfg.ID, inst.cfg.CgroupParent + if cgroupParent == "" { + return nil, errgrpc.ToGRPCf(errdefs.ErrNotFound, "sandbox %q has no cgroup parent", id) + } + cg, err := cgroupsv2.Load(cgroupParent) + if err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to load sandbox cgroup %q", cgroupParent) + } + stats, err := cg.StatFiltered(cgroupsv2.StatCPU | cgroupsv2.StatMemory) + if err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to read sandbox cgroup %q", cgroupParent) + } + data, err := typeurl.MarshalAnyToProto(stats) + if err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to marshal sandbox metrics") + } + return &api.SandboxMetricsResponse{Metrics: &types.Metric{ + Timestamp: protobuf.ToTimestamp(time.Now()), + ID: id, + Data: data, + }}, nil +} + +// UpdateSandbox is not implemented: the sandbox has no resources of its own. +func (s *Service) UpdateSandbox(ctx context.Context, r *api.UpdateSandboxRequest) (*api.UpdateSandboxResponse, error) { + return nil, errgrpc.ToGRPCf(errdefs.ErrNotImplemented, "%s does not implement UpdateSandbox", BinaryName) +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service_test.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service_test.go new file mode 100644 index 0000000000..392a4b8c49 --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service_test.go @@ -0,0 +1,113 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "testing" + "time" + + api "github.com/containerd/containerd/api/runtime/sandbox/v1" + "github.com/containerd/errdefs" + "github.com/containerd/errdefs/pkg/errgrpc" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/containerd/containerd/v2/pkg/shutdown" +) + +func TestServiceWithoutSandbox(t *testing.T) { + ctx := context.Background() + _, sd := shutdown.WithShutdown(ctx) + s := NewService(sd) + + // Every sandbox scoped call is NotFound until a sandbox is created. + _, err := s.StartSandbox(ctx, &api.StartSandboxRequest{SandboxID: testID}) + assert.True(t, errdefs.IsNotFound(errgrpc.ToNative(err)), "start: %v", err) + _, err = s.StopSandbox(ctx, &api.StopSandboxRequest{SandboxID: testID}) + assert.True(t, errdefs.IsNotFound(errgrpc.ToNative(err)), "stop: %v", err) + _, err = s.SandboxStatus(ctx, &api.SandboxStatusRequest{SandboxID: testID}) + assert.True(t, errdefs.IsNotFound(errgrpc.ToNative(err)), "status: %v", err) + _, err = s.PingSandbox(ctx, &api.PingRequest{SandboxID: testID}) + assert.True(t, errdefs.IsNotFound(errgrpc.ToNative(err)), "ping: %v", err) + _, err = s.WaitSandbox(ctx, &api.WaitSandboxRequest{SandboxID: testID}) + assert.True(t, errdefs.IsNotFound(errgrpc.ToNative(err)), "wait: %v", err) + _, err = s.SandboxMetrics(ctx, &api.SandboxMetricsRequest{SandboxID: testID}) + assert.True(t, errdefs.IsNotFound(errgrpc.ToNative(err)), "metrics: %v", err) + + _, err = s.UpdateSandbox(ctx, &api.UpdateSandboxRequest{SandboxID: testID}) + assert.True(t, errdefs.IsNotImplemented(errgrpc.ToNative(err)), "update: %v", err) + + resp, err := s.Platform(ctx, &api.PlatformRequest{SandboxID: testID}) + require.NoError(t, err) + assert.Equal(t, "linux", resp.GetPlatform().GetOS()) + + // Shutting down, even an unknown sandbox, ends the shim. + select { + case <-sd.Done(): + t.Fatal("the shim must not shut down before ShutdownSandbox") + default: + } + _, err = s.ShutdownSandbox(ctx, &api.ShutdownSandboxRequest{SandboxID: testID}) + require.NoError(t, err) + select { + case <-sd.Done(): + case <-time.After(5 * time.Second): + t.Fatal("ShutdownSandbox must end the shim") + } +} + +func TestSandboxStatusBeforeStart(t *testing.T) { + ctx := context.Background() + _, sd := shutdown.WithShutdown(ctx) + s := NewService(sd) + s.sandbox = &instance{cfg: &Config{ID: testID}, done: make(chan struct{})} + + // A created sandbox is not ready until it is started. + status, err := s.SandboxStatus(ctx, &api.SandboxStatusRequest{SandboxID: testID}) + require.NoError(t, err) + assert.Equal(t, runtime.PodSandboxState_SANDBOX_NOTREADY.String(), status.GetState()) + assert.Nil(t, status.GetExitedAt()) + + _, err = s.StartSandbox(ctx, &api.StartSandboxRequest{SandboxID: testID}) + require.NoError(t, err) + status, err = s.SandboxStatus(ctx, &api.SandboxStatusRequest{SandboxID: testID}) + require.NoError(t, err) + assert.Equal(t, runtime.PodSandboxState_SANDBOX_READY.String(), status.GetState()) +} + +func TestShutdownSandboxOther(t *testing.T) { + ctx := context.Background() + _, sd := shutdown.WithShutdown(ctx) + s := NewService(sd) + s.sandbox = &instance{cfg: &Config{ID: testID}, done: make(chan struct{})} + + // A shutdown meant for another sandbox must neither stop the hosted one + // nor end the shim. + _, err := s.ShutdownSandbox(ctx, &api.ShutdownSandboxRequest{SandboxID: "other"}) + assert.True(t, errdefs.IsNotFound(errgrpc.ToNative(err)), "shutdown: %v", err) + require.NotNil(t, s.sandbox) + assert.False(t, s.sandbox.stopped) + select { + case <-sd.Done(): + t.Fatal("the shim must not shut down for another sandbox") + default: + } +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/task.go b/cmd/containerd-shim-sandboxed-runc-v2/task.go new file mode 100644 index 0000000000..d404399d12 --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/task.go @@ -0,0 +1,100 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + + taskAPI "github.com/containerd/containerd/api/runtime/task/v3" + "github.com/containerd/plugin" + "github.com/containerd/plugin/registry" + "github.com/containerd/ttrpc" + + "github.com/containerd/containerd/v2/cmd/containerd-shim-runc-v2/task" + ptypes "github.com/containerd/containerd/v2/pkg/protobuf/types" + "github.com/containerd/containerd/v2/pkg/shim" + "github.com/containerd/containerd/v2/pkg/shutdown" + "github.com/containerd/containerd/v2/plugins" +) + +// The containers of the pod run through the task service of +// containerd-shim-runc-v2, the very same implementation: this file only wraps +// it. Nothing about how containers are created, started, exec'd or deleted +// changes. +// +// The one thing the wrapper overrides is who ends the shim. containerd sends +// the task Shutdown RPC after every task deletion, and the shared task +// service exits the shim as soon as no container is left. That is right for a +// shim that exists for its tasks and wrong for a shim whose sandbox outlives +// them: a pod with no containers is still a pod. Here Shutdown is +// acknowledged and ignored; the shim exits when its sandbox is shut down +// (see the sandbox package). +// +// TODO: consolidate with containerd-shim-runc-v2 so that both shims register +// the same task service without a wrapper, e.g. by letting the host of the +// shared service decide the shim lifetime. Wrapping the shutdown.Service the +// shared service receives (a no-op Shutdown, everything else delegated) would +// work as well; the RPC wrapper is kept because it states the intent in one +// place. + +func init() { + registry.Register(&plugin.Registration{ + Type: plugins.TTRPCPlugin, + ID: "task", + Requires: []plugin.Type{ + plugins.EventPlugin, + plugins.InternalPlugin, + }, + InitFn: func(ic *plugin.InitContext) (any, error) { + pp, err := ic.GetByID(plugins.EventPlugin, "publisher") + if err != nil { + return nil, err + } + ss, err := ic.GetByID(plugins.InternalPlugin, "shutdown") + if err != nil { + return nil, err + } + svc, err := task.NewTaskService(ic.Context, pp.(shim.Publisher), ss.(shutdown.Service)) + if err != nil { + return nil, err + } + return &taskService{TTRPCTaskService: svc}, nil + }, + }) +} + +// taskService is the task service of containerd-shim-runc-v2 with the shim +// lifetime taken out of its hands. Every RPC but Shutdown is the embedded one. +type taskService struct { + taskAPI.TTRPCTaskService +} + +var _ shim.TTRPCService = (*taskService)(nil) + +// RegisterTTRPC implements shim.TTRPCService. +func (s *taskService) RegisterTTRPC(server *ttrpc.Server) error { + taskAPI.RegisterTTRPCTaskService(server, s) + return nil +} + +// Shutdown acknowledges the request and keeps the shim running: the sandbox +// decides when the shim exits. +func (s *taskService) Shutdown(context.Context, *taskAPI.ShutdownRequest) (*ptypes.Empty, error) { + return &ptypes.Empty{}, nil +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/task_test.go b/cmd/containerd-shim-sandboxed-runc-v2/task_test.go new file mode 100644 index 0000000000..a17b71e1df --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/task_test.go @@ -0,0 +1,53 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + "testing" + + taskAPI "github.com/containerd/containerd/api/runtime/task/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ptypes "github.com/containerd/containerd/v2/pkg/protobuf/types" +) + +// shuttingDown counts the Shutdown calls that reach the wrapped task service. +type shuttingDown struct { + taskAPI.TTRPCTaskService + calls int +} + +func (s *shuttingDown) Shutdown(context.Context, *taskAPI.ShutdownRequest) (*ptypes.Empty, error) { + s.calls++ + return &ptypes.Empty{}, nil +} + +func TestShutdownKeepsTheShimRunning(t *testing.T) { + inner := &shuttingDown{} + svc := &taskService{TTRPCTaskService: inner} + + // containerd sends Shutdown after every task deletion; the shared task + // service would exit the shim once no container is left. The sandbox + // shim must stay up for its pod. + _, err := svc.Shutdown(context.Background(), &taskAPI.ShutdownRequest{ID: "container"}) + require.NoError(t, err) + assert.Equal(t, 0, inner.calls, "the shared task service must not see the shutdown") +} From da144993d55caa04d7357ada139f8b393bf93842 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Mon, 2 Nov 2026 09:46:22 -0800 Subject: [PATCH 07/13] ci: run critest through the sandboxed shim critest.sh takes SANDBOXER, the sandbox controller of the runtime handler (podsandbox by default; shim adds disable_pause_image_pull, since a Sandbox API shim needs no pause image). The Linux integration job runs the critest suite a second time on the runc.v2 legs, with TEST_RUNTIME_PATH pointing at containerd-shim-sandboxed-runc-v2 and the shim sandboxer. The one spec that needs a shared pod PID namespace is skipped by name until that lands; the user namespace specs skip themselves, since the handler does not advertise user namespaces. Signed-off-by: Maksym Pavlenko --- .github/workflows/ci.yml | 24 ++++++++++++++++++++++++ script/critest.sh | 12 ++++++++++++ 2 files changed, 36 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c73f51d39..084560e669 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -607,6 +607,28 @@ jobs: env sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report" + # Experimental: the same critest suite through the pause-less sandbox + # shim (containerd-shim-sandboxed-runc-v2 selected with runtime_path, + # with the shim sandboxer). Specs that need what the shim does not + # support yet are skipped by their full name; the list shrinks as the + # features land: + # - "runtime should support PodPID": a shared pod PID namespace needs + # a durable PID 1 (sandbox-init). + # The user namespace specs skip themselves: the handler does not + # advertise user namespaces until the shim supports them. + # The sandboxed shim is a Go shim: it runs on the runc.v2 legs only. + - if: matrix.runtime == 'io.containerd.runc.v2' + name: cri-tools critest (sandbox shim) + env: + TEST_RUNTIME: ${{ matrix.runtime }} + TEST_RUNTIME_PATH: /usr/local/bin/containerd-shim-sandboxed-runc-v2 + SANDBOXER: shim + CGROUP_DRIVER: ${{ matrix.cgroup_driver }} + SKIP_TEST: "NamespaceOption runtime should support PodPID" + run: | + env + sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report-sandbox-shim" + # The Rust shim does not implement checkpoint yet. - if: matrix.os != 'ubuntu-24.04-arm' && matrix.runtime != 'io.containerd.runc.v2-rs' name: Checkpoint/Restore Disable via CRI @@ -641,6 +663,8 @@ jobs: *-gotest.json ${{github.workspace}}/report/*.xml ${{github.workspace}}/report/*.log + ${{github.workspace}}/report-sandbox-shim/*.xml + ${{github.workspace}}/report-sandbox-shim/*.log integration-vm: name: VM integration diff --git a/script/critest.sh b/script/critest.sh index dc944a2ae9..27f8ff8af6 100755 --- a/script/critest.sh +++ b/script/critest.sh @@ -50,6 +50,17 @@ function cleanup() { trap cleanup EXIT mkdir -p ${BDIR}/{root,state} + +# SANDBOXER is the sandbox controller of the runtime handler: "podsandbox" +# (the pause container, the default) or "shim" (the Sandbox API of the shim, +# which needs no pause image). +SANDBOXER="${SANDBOXER:-podsandbox}" +SANDBOX_CONFIG="sandboxer = \"${SANDBOXER}\"" +if [ "${SANDBOXER}" = "shim" ]; then + SANDBOX_CONFIG="${SANDBOX_CONFIG} +disable_pause_image_pull = true" +fi + containerd_config="${CONTAINERD_CONFIG_FILE:-${BDIR}/config.toml}" if [[ -z "${CONTAINERD_CONFIG_FILE:-}" ]]; then cat > "${containerd_config}" < "snapshotter \"overlayfs\" doesn't support idmap mounts on this host, From d113e117d8a99d40c0a3eb68674e3d5b5ef8a0d9 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Tue, 8 Sep 2026 21:14:21 -0700 Subject: [PATCH 08/13] sandbox: tolerate a shim exiting during ShutdownSandbox A sandbox shim may exit as soon as it has shut its sandbox down, before containerd reads the ShutdownSandbox response, in which case the client sees ttrpc.ErrClosed. The task Shutdown RPC has the same race and shimTask.Shutdown treats a closed connection as a successful shutdown; controllerLocal.Shutdown returned it as an error instead, failing RemovePodSandbox for a sandbox that was already gone. Signed-off-by: Maksym Pavlenko --- plugins/sandbox/controller.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/sandbox/controller.go b/plugins/sandbox/controller.go index 52fa8caf54..1d237a2560 100644 --- a/plugins/sandbox/controller.go +++ b/plugins/sandbox/controller.go @@ -18,6 +18,7 @@ package sandbox import ( "context" + "errors" "fmt" "os" "time" @@ -27,6 +28,7 @@ import ( "github.com/containerd/log" "github.com/containerd/plugin" "github.com/containerd/plugin/registry" + "github.com/containerd/ttrpc" "github.com/containerd/typeurl/v2" imagespec "github.com/opencontainers/image-spec/specs-go/v1" @@ -260,8 +262,11 @@ func (c *controllerLocal) Shutdown(ctx context.Context, sandboxID string) error return err } + // A shim may exit as soon as it has shut its sandbox down, before the + // response is read; a closed connection means the shutdown happened, as + // for the task Shutdown RPC. _, err = svc.ShutdownSandbox(ctx, &runtimeAPI.ShutdownSandboxRequest{SandboxID: sandboxID}) - if err != nil { + if err != nil && !errors.Is(err, ttrpc.ErrClosed) { return fmt.Errorf("failed to shutdown sandbox: %w", errgrpc.ToNative(err)) } From 16d7ff4f1ac405d89f6a67e3da00f1f0940b26a9 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Fri, 23 Oct 2026 13:24:49 -0700 Subject: [PATCH 09/13] cmd: add containerd-sandbox-init A minimal PID 1 for the PID namespace of a pod that shares it, run by containerd-shim-sandboxed-runc-v2 in place of a pause container. It stays alive so that the namespace does, ignores SIGCHLD so that the kernel reaps the orphans reparented to it, and removes every signal handler so that nothing sent from inside the pod can end it or run code in it: the kernel drops a signal sent to the init of a PID namespace when init has no handler for it. The Go runtime installs handlers for almost every signal, including the synchronous faults, SIGPROF and the signals 32 and 33 that os/signal cannot reset, so those go back to the default disposition, or are ignored for the others, through rt_sigaction. Only SIGKILL from the parent namespace ends it. It makes itself non-dumpable, so that the other processes of the pod can neither ptrace it nor open its /proc entries without CAP_SYS_PTRACE. It runs with one P, no garbage collector and parks in pause(2). It is built static like the shims and installed with them. Signed-off-by: Maksym Pavlenko --- Makefile | 4 + Makefile.linux | 1 + cmd/containerd-sandbox-init/main.go | 134 ++++++++++++++++++ cmd/containerd-sandbox-init/main_test.go | 164 +++++++++++++++++++++++ 4 files changed, 303 insertions(+) create mode 100644 cmd/containerd-sandbox-init/main.go create mode 100644 cmd/containerd-sandbox-init/main_test.go diff --git a/Makefile b/Makefile index f6e0c71993..6b7c423d29 100644 --- a/Makefile +++ b/Makefile @@ -276,6 +276,10 @@ bin/containerd-shim-sandboxed-runc-v2: cmd/containerd-shim-sandboxed-runc-v2 FOR @echo "$(WHALE) $@" CGO_ENABLED=${SHIM_CGO_ENABLED} $(GO) build ${GO_BUILD_FLAGS} -o $@ ${SHIM_GO_LDFLAGS} ${SHIM_GO_TAGS} ./cmd/containerd-shim-sandboxed-runc-v2 +bin/containerd-sandbox-init: cmd/containerd-sandbox-init FORCE # PID 1 of a shared pod PID namespace for the sandboxed shim, static like the shims + @echo "$(WHALE) $@" + CGO_ENABLED=${SHIM_CGO_ENABLED} $(GO) build ${GO_BUILD_FLAGS} -o $@ ${SHIM_GO_LDFLAGS} ${SHIM_GO_TAGS} ./cmd/containerd-sandbox-init + binaries: $(BINARIES) ## build binaries @echo "$(WHALE) $@" diff --git a/Makefile.linux b/Makefile.linux index 8b9750aebe..6f50624298 100644 --- a/Makefile.linux +++ b/Makefile.linux @@ -18,6 +18,7 @@ WHALE="+" ONI="-" COMMANDS += containerd-shim-runc-v2 COMMANDS += containerd-shim-sandboxed-runc-v2 +COMMANDS += containerd-sandbox-init # check GOOS for cross compile builds ifeq ($(GOOS),linux) diff --git a/cmd/containerd-sandbox-init/main.go b/cmd/containerd-sandbox-init/main.go new file mode 100644 index 0000000000..4b2a951c58 --- /dev/null +++ b/cmd/containerd-sandbox-init/main.go @@ -0,0 +1,134 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// containerd-sandbox-init is PID 1 of the PID namespace of a pod that shares +// it (shareProcessNamespace). containerd-shim-sandboxed-runc-v2, the +// pause-less sandbox shim, starts it through runc for those pods only, and it +// runs as a child of the shim. An ordinary pod, one that does not share its +// PID namespace, has no sandbox process at all. +// +// It does what PID 1 of a pod has to do and nothing else: it stays alive so +// that the namespace does, it lets the kernel reap the orphans that get +// reparented to it, and it cannot be ended from inside the pod. +// +// The kernel delivers a signal to the init of a PID namespace, from inside +// that namespace, only when init has a handler installed for it, and discards +// it otherwise. SIGKILL and SIGSTOP reach it from the parent namespace only, +// and SIGKILL from the shim is how it ends. The Go runtime installs handlers +// for almost every signal and ends the process on most of them. A Go init +// that kept those handlers could be killed from a member container with +// kill -HUP 1 or kill -SEGV 1, so disableSignals removes every one of them. +// +// It is also not dumpable, so that the other processes of the pod cannot +// inspect or control it: without CAP_SYS_PTRACE they can neither ptrace it +// nor open /proc/1/root, /proc/1/mem and the like. +package main + +import ( + "os" + "os/signal" + "runtime" + "runtime/debug" + "syscall" + "unsafe" + + "golang.org/x/sys/unix" +) + +func main() { + run() +} + +// run sets the process up and parks it; it never returns. +func run() { + // runc makes the process non-dumpable, but execve made it dumpable + // again, since it changed no credentials. + if err := unix.Prctl(unix.PR_SET_DUMPABLE, 0, 0, 0, 0); err != nil { + os.Exit(1) + } + // A single scheduler thread (GOMAXPROCS) and no garbage collector: + // there is nothing to run and nothing to collect. Like pause, the + // process then sleeps until it is killed. + runtime.GOMAXPROCS(1) + debug.SetGCPercent(-1) + if err := disableSignals(); err != nil { + os.Exit(1) + } + debug.FreeOSMemory() + for { + // A blocking system call rather than select {}: with no other + // goroutine the runtime would report a deadlock. + unix.Pause() + } +} + +// disableSignals leaves no signal handler in place, so that no signal sent +// from inside the pod runs any code in the process. SIGCHLD is ignored rather +// than defaulted so that the kernel reaps the children reparented to PID 1 on +// its own, with no wait loop. Ignoring SIGURG gives up asynchronous +// preemption, which a process whose only goroutine waits in pause(2) does not +// need. os/signal cannot touch the synchronous faults, for which the runtime +// installs handlers that exit the process when the signal came from kill(2), +// nor SIGPROF, whose handler it keeps for profiling, nor signals 32 and 33, +// which the C library reserves and the runtime keeps for itself (33 for +// system calls that must run on every thread, which this process never +// makes): the faults go back to the kernel default and the others are +// ignored, through rt_sigaction. +func disableSignals() error { + // Signals 32 and 33, reserved by the C library and kept by the Go runtime. + const ( + sigRuntime1 = syscall.Signal(32) + sigRuntime2 = syscall.Signal(33) + ) + for n := 1; n <= 64; n++ { + sig := syscall.Signal(n) + switch sig { + case unix.SIGKILL, unix.SIGSTOP: + // Uncatchable, and how the shim ends the process. + case unix.SIGILL, unix.SIGTRAP, unix.SIGBUS, unix.SIGFPE, unix.SIGSEGV, unix.SIGSTKFLT, unix.SIGSYS: + if err := setAction(sig, sigDFL); err != nil { + return err + } + case unix.SIGPROF, sigRuntime1, sigRuntime2: + if err := setAction(sig, sigIGN); err != nil { + return err + } + default: + signal.Ignore(sig) + } + } + return nil +} + +// The dispositions setAction sets. +const ( + sigDFL = 0 + sigIGN = 1 +) + +// setAction sets the disposition of sig, with no flags and an empty mask. The +// struct sigaction passed is sized for every architecture and starts with the +// handler, as it does on every supported one (MIPS puts the flags first and +// has a larger sigset); the last argument is the size of the kernel sigset. +func setAction(sig syscall.Signal, handler uintptr) error { + act := [4]uint64{uint64(handler)} + if _, _, errno := unix.RawSyscall6(unix.SYS_RT_SIGACTION, uintptr(sig), uintptr(unsafe.Pointer(&act)), 0, 8, 0, 0); errno != 0 { + return errno + } + return nil +} diff --git a/cmd/containerd-sandbox-init/main_test.go b/cmd/containerd-sandbox-init/main_test.go new file mode 100644 index 0000000000..dc0a656583 --- /dev/null +++ b/cmd/containerd-sandbox-init/main_test.go @@ -0,0 +1,164 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" + + "github.com/containerd/containerd/v2/pkg/testutil" +) + +// childEnv makes the test binary run as the init instead of the tests. +const childEnv = "CONTAINERD_SANDBOX_INIT_TEST_CHILD" + +func TestMain(m *testing.M) { + if os.Getenv(childEnv) == "1" { + run() + } + os.Exit(m.Run()) +} + +// initUser is the user the shim runs the init as. +const initUser = 65535 + +// TestSignals runs the init as PID 1 of a new PID namespace, as the user the +// shim runs it as, and sends it every signal but SIGKILL and SIGSTOP, from +// the parent namespace: a signal that ended it here would end it from inside +// the pod as well (see the package doc). +func TestSignals(t *testing.T) { + testutil.RequiresRoot(t) + + cmd := exec.Command(initCopy(t)) + cmd.Env = append(os.Environ(), childEnv+"=1") + cmd.SysProcAttr = &syscall.SysProcAttr{ + Cloneflags: unix.CLONE_NEWPID, + Credential: &syscall.Credential{Uid: initUser, Gid: initUser}, + } + require.NoError(t, cmd.Start()) + // waitErr is valid once exited is closed. + var waitErr error + exited := make(chan struct{}) + go func() { + waitErr = cmd.Wait() + close(exited) + }() + t.Cleanup(func() { + _ = cmd.Process.Kill() + <-exited + }) + pid := cmd.Process.Pid + + // The setup is done once the last signal of the loop, 64, is ignored. + require.Eventually(t, func() bool { + ignored, _ := signalMasks(t, pid) + return ignored&(1<<63) != 0 + }, 10*time.Second, 10*time.Millisecond, "the init did not finish its signal setup") + + ignored, caught := signalMasks(t, pid) + for _, sig := range []syscall.Signal{unix.SIGCHLD, unix.SIGURG, unix.SIGPROF} { + assert.NotZero(t, ignored&(1<<(sig-1)), "%s must be ignored", unix.SignalName(sig)) + } + assert.Zero(t, caught, "no signal may have a handler (SigCgt %#x)", caught) + + // The /proc/ entries of a process that is not dumpable belong to + // root (the directory itself keeps the owner of the process). + var st unix.Stat_t + require.NoError(t, unix.Stat(fmt.Sprintf("/proc/%d/status", pid), &st)) + assert.EqualValues(t, 0, st.Uid, "the init must not be dumpable") + + for n := 1; n <= 64; n++ { + sig := syscall.Signal(n) + if sig == unix.SIGKILL || sig == unix.SIGSTOP { + continue + } + require.NoError(t, unix.Kill(pid, sig), "signal %d", n) + select { + case <-exited: + t.Fatalf("signal %d (%s) ended the init: %v", n, unix.SignalName(sig), waitErr) + case <-time.After(20 * time.Millisecond): + } + } + + require.NoError(t, unix.Kill(pid, unix.SIGKILL)) + select { + case <-exited: + var exitErr *exec.ExitError + require.ErrorAs(t, waitErr, &exitErr) + assert.Equal(t, unix.SIGKILL, exitErr.Sys().(syscall.WaitStatus).Signal()) + case <-time.After(5 * time.Second): + t.Fatal("SIGKILL must end the init") + } +} + +// initCopy copies the test binary where the init user can run it: the +// directories go test builds it in are not traversable for other users. +func initCopy(t *testing.T) string { + self, err := os.Executable() + require.NoError(t, err) + dir := t.TempDir() + for d := dir; d != os.TempDir(); d = filepath.Dir(d) { + require.NoError(t, os.Chmod(d, 0o755)) + } + src, err := os.Open(self) + require.NoError(t, err) + defer src.Close() + path := filepath.Join(dir, "init") + dst, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0o755) + require.NoError(t, err) + _, err = io.Copy(dst, src) + require.NoError(t, errors.Join(err, dst.Close())) + return path +} + +// signalMasks reads the ignored and caught signal sets of pid from /proc; +// bit n-1 stands for signal n. +func signalMasks(t *testing.T, pid int) (ignored, caught uint64) { + data, err := os.ReadFile(fmt.Sprintf("/proc/%d/status", pid)) + require.NoError(t, err) + for _, line := range strings.Split(string(data), "\n") { + field, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + value = strings.TrimSpace(value) + switch field { + case "SigIgn": + ignored, err = strconv.ParseUint(value, 16, 64) + require.NoError(t, err) + case "SigCgt": + caught, err = strconv.ParseUint(value, 16, 64) + require.NoError(t, err) + } + } + return ignored, caught +} From e09b89ff81abe92fb38e4a60cd7a16004f69da4f Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Fri, 23 Oct 2026 13:24:50 -0700 Subject: [PATCH 10/13] cmd/containerd-shim-sandboxed-runc-v2: persist the runc options in the sandbox bundle "shim start" writes the runc options of the runtime handler and the runtime name into the sandbox bundle, the files runc.NewContainer writes for a task bundle. The sandbox service reads them to run a process through the same runc, root and cgroup driver as the containers of the pod, and "shim delete" reads them to force-delete that runc container once the shim is gone. Signed-off-by: Maksym Pavlenko --- .../manager.go | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/cmd/containerd-shim-sandboxed-runc-v2/manager.go b/cmd/containerd-shim-sandboxed-runc-v2/manager.go index 114999311d..cb9632de6d 100644 --- a/cmd/containerd-shim-sandboxed-runc-v2/manager.go +++ b/cmd/containerd-shim-sandboxed-runc-v2/manager.go @@ -41,6 +41,7 @@ import ( "github.com/opencontainers/runtime-spec/specs-go/features" legacymanager "github.com/containerd/containerd/v2/cmd/containerd-shim-runc-v2/manager" + "github.com/containerd/containerd/v2/cmd/containerd-shim-runc-v2/runc" "github.com/containerd/containerd/v2/cmd/containerd-shim-sandboxed-runc-v2/sandbox" "github.com/containerd/containerd/v2/defaults" "github.com/containerd/containerd/v2/pkg/namespaces" @@ -163,6 +164,12 @@ func (m *manager) Start(ctx context.Context, opts *bootapi.BootstrapParams) (_ * return nil, fmt.Errorf("%s requires cgroup v2 (the unified hierarchy) and this host runs cgroup v1: %w", sandbox.BinaryName, errdefs.ErrNotImplemented) } + // The working directory is the sandbox bundle. + cwd, err := os.Getwd() + if err != nil { + return nil, err + } + var params bootapi.BootstrapResult params.Version = 3 params.Protocol = "ttrpc" @@ -238,6 +245,16 @@ func (m *manager) Start(ctx context.Context, opts *bootapi.BootstrapParams) (_ * if found, err := opts.FindExtension(&runcOpts); err != nil { return nil, fmt.Errorf("failed to fetch runc options: %w", err) } else if found { + // The runc options are persisted in the bundle. The sandbox service + // reads them to run containerd-sandbox-init through the same runc, + // root and cgroup driver as the containers of the pod, and Stop + // reads them to delete that container once the shim is gone. + if err := runc.WriteOptions(cwd, &runcOpts); err != nil { + return nil, fmt.Errorf("failed to persist the runc options: %w", err) + } + if err := runc.WriteRuntime(cwd, runcOpts.GetBinaryName()); err != nil { + return nil, fmt.Errorf("failed to persist the runtime name: %w", err) + } if shimCgroup := runcOpts.GetShimCgroup(); shimCgroup != "" { cg, err := cgroupsv2.Load(shimCgroup) if err != nil { @@ -259,11 +276,14 @@ func (m *manager) Start(ctx context.Context, opts *bootapi.BootstrapParams) (_ * // Stop is "shim delete". containerd runs it in a bundle once the shim that // served the bundle is gone: in the sandbox bundle, and in the bundle of every -// container task that joined the shim. Nothing needs telling apart: the runc -// task cleanup of containerd-shim-runc-v2 (force-delete the runc container of -// that id, detach its rootfs) does nothing in the sandbox bundle, and the -// sandbox cleanup (detach every mount under the bundle, remove the pod files) -// does nothing in a task bundle, so both run every time. +// container task that joined the shim. Stop does not tell the two apart and +// runs both cleanups every time: +// - The runc task cleanup of containerd-shim-runc-v2 force-deletes the runc +// container of that id and detaches its rootfs. In the sandbox bundle of a +// shared-PID pod that deletes containerd-sandbox-init; in any other +// sandbox bundle it does nothing. +// - The sandbox cleanup detaches every mount under the bundle and removes +// the pod files. In a task bundle it does nothing. func (m *manager) Stop(ctx context.Context, id string) (shim.StopStatus, error) { status, err := m.legacy.Stop(ctx, id) cwd, cwdErr := os.Getwd() From 96a675454249863ea514d58d7d0fdc11db678760 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Fri, 23 Oct 2026 13:24:50 -0700 Subject: [PATCH 11/13] sandbox: share a pod PID namespace through containerd-sandbox-init A pod that shares its PID namespace (shareProcessNamespace) needs a durable PID 1 for it, the role pause plays. CreateSandbox now runs containerd-sandbox-init for such a pod through runc: in the sandbox bundle with a tmpfs rootfs holding only the program, under the pod cgroup with either cgroup driver, in the pod network, IPC and UTS namespaces, as an unprivileged user without capabilities, under containerd's default seccomp profile, with the masked and read-only /proc paths of a default container spec and the OOM score and CPU shares of a pause container. Its container id is the sandbox id. The PID namespace is pinned under the bundle like the IPC and UTS namespaces and listed in the sandbox spec, so the containers join it the same way; the sandbox still reports pid 0. StartSandbox execs the program, StopSandbox kills and deletes it and reports its exit, and an init that exits on its own stops the sandbox. Every other pod still has no sandbox process. The pod SELinux label is not applied to the init yet. Signed-off-by: Maksym Pavlenko --- .../sandbox/bundle.go | 16 +- .../sandbox/config.go | 71 ++-- .../sandbox/config_test.go | 19 +- .../sandbox/doc.go | 13 +- .../sandbox/init.go | 328 ++++++++++++++++++ .../sandbox/init_test.go | 202 +++++++++++ .../sandbox/service.go | 129 +++++-- .../sandbox/service_test.go | 29 ++ 8 files changed, 741 insertions(+), 66 deletions(-) create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/sandbox/init.go create mode 100644 cmd/containerd-shim-sandboxed-runc-v2/sandbox/init_test.go diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle.go index c2373a1401..206c312b6c 100644 --- a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle.go +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle.go @@ -37,11 +37,13 @@ import ( // Everything the sandbox creates lives under the shim bundle at fixed paths // (see the package doc): // -// /ns/{ipc,uts} namespace pins +// /ns/{ipc,uts} namespace pins, and ns/pid for a shared-PID pod // /hostname the pod hostname // /hosts a copy of the host /etc/hosts // /resolv.conf the pod DNS configuration // /shm the pod /dev/shm tmpfs +// /rootfs the tmpfs rootfs of containerd-sandbox-init, with +// its config.json and init.pid beside (shared-PID pods) // setupFiles creates the pod shared files and returns the mounts that expose // them to the containers of the pod; CRI picks the sources up from the @@ -105,16 +107,20 @@ func copyFile(src, dst string) error { } // Cleanup removes everything the sandbox shim created under bundle: it -// detaches the mounts at their fixed paths (the namespace pins and the shm -// tmpfs) and removes the pin and shared files. It is idempotent and relies on -// no state, so it serves create failure, StopSandbox and "shim delete" alike; -// a mount left behind would keep containerd from deleting the bundle. +// detaches the mounts at their fixed paths (the namespace pins, the shm tmpfs +// and the init rootfs) and removes the pin and shared files. It is idempotent +// and relies on no state, so it serves create failure, StopSandbox and "shim +// delete" alike; a mount left behind would keep containerd from deleting the +// bundle. The init container itself is deleted through runc by its owner +// (initProcess.stop, or the runc task cleanup of "shim delete"). func Cleanup(ctx context.Context, bundle string) error { var errs []error for _, p := range []string{ filepath.Join(bundle, pinDir, ipcPin), filepath.Join(bundle, pinDir, utsPin), + filepath.Join(bundle, pinDir, pidPin), filepath.Join(bundle, sandboxfiles.ShmDir), + filepath.Join(bundle, initRootfs), } { // Only a mount point is unmounted: umount(2) on a plain file needs // the same privilege as on a mount, and answers EPERM before EINVAL. diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config.go index 94e5c0af57..bba2c7637e 100644 --- a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config.go +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config.go @@ -47,16 +47,18 @@ type Config struct { // HostPID means the containers of the pod use the host PID namespace. HostPID bool // SharedPID means the pod asked for one PID namespace shared by its - // containers, which this shim cannot provide yet (see buildSpec). + // containers (shareProcessNamespace); containerd-sandbox-init is then + // run as its PID 1 (see initProcess). SharedPID bool // NetNSPath is the network namespace CRI created for the pod. NetNSPath string // Sysctls is the effective set of namespaced sysctls to apply. Sysctls map[string]string - // CgroupParent is the pod cgroup as a cgroup v2 group path, used for - // metrics. kubelet spells it that way with both cgroup drivers: the - // systemd driver hands over the expanded slice path, which CRI shortens - // to its base for runc. + // CgroupParent is the pod cgroup as a cgroup v2 group path: the sandbox + // reports its metrics, and places the init of a shared-PID pod under it. + // kubelet spells it that way with both cgroup drivers: the systemd + // driver hands over the expanded slice path, which CRI shortens to its + // base for runc. CgroupParent string // DNS is the pod DNS configuration; nil means the host resolv.conf. DNS *runtime.DNSConfig @@ -236,40 +238,51 @@ func sysctlPath(key string) (path, dotted string, err error) { } // buildSpec synthesizes the sandbox spec CRI and NRI consume. It is not the -// spec of a process (there is none): it describes the namespaces the sandbox -// holds, from what was actually pinned, the sysctls applied to them, and the -// pod shared files as mounts. A namespace the pod shares with the host has no -// entry. Linux is always set, which is how CRI tells a sandbox without a -// process from an older Sandbox API shim that returns no spec. +// spec of a process: it describes the namespaces the sandbox holds, from what +// was actually pinned, the sysctls applied to them, and the pod shared files +// as mounts. A namespace the pod shares with the host has no entry. Linux is +// always set, which is how CRI tells a sandbox without a process from an +// older Sandbox API shim that returns no spec. // -// The PID namespace entry has no path: the sandbox holds none. With a pause -// container the pod PID namespace is the one pause is PID 1 of, and a pod -// that asks to share it (NamespaceMode_POD, the CRI default and kubelet's -// shareProcessNamespace) joins its containers to it. Holding a shared PID -// namespace needs a durable PID 1 (sandbox-init, a follow-up); until then -// the entry declares that each container gets a new PID namespace of its -// own, and CRI honors that declaration instead of failing. -func buildSpec(cfg *Config, pins Pins, mounts []specs.Mount) *specs.Spec { +// The PID namespace entry has a path only for a pod that shares its PID +// namespace (NamespaceMode_POD, the CRI default and kubelet's +// shareProcessNamespace): pidNS, the pinned namespace containerd-sandbox-init +// is PID 1 of. Otherwise the sandbox holds no PID namespace, and the entry, without a +// path, declares that each container gets a new one of its own; CRI honors +// the declaration. A host PID pod has no entry. +func buildSpec(cfg *Config, pins Pins, mounts []specs.Mount, pidNS string) *specs.Spec { s := &specs.Spec{ Version: specs.Version, Hostname: cfg.Hostname, Annotations: cfg.Annotations, Mounts: mounts, Linux: &specs.Linux{ - Sysctl: cfg.Sysctls, + Sysctl: cfg.Sysctls, + Namespaces: podNamespaces(cfg, pins), }, } - if cfg.NetNSPath != "" { - s.Linux.Namespaces = append(s.Linux.Namespaces, specs.LinuxNamespace{Type: specs.NetworkNamespace, Path: cfg.NetNSPath}) - } - if pins.IPC != "" { - s.Linux.Namespaces = append(s.Linux.Namespaces, specs.LinuxNamespace{Type: specs.IPCNamespace, Path: pins.IPC}) - } - if pins.UTS != "" { - s.Linux.Namespaces = append(s.Linux.Namespaces, specs.LinuxNamespace{Type: specs.UTSNamespace, Path: pins.UTS}) - } - if !cfg.HostPID { + switch { + case cfg.HostPID: + case pidNS != "": + s.Linux.Namespaces = append(s.Linux.Namespaces, specs.LinuxNamespace{Type: specs.PIDNamespace, Path: pidNS}) + default: s.Linux.Namespaces = append(s.Linux.Namespaces, specs.LinuxNamespace{Type: specs.PIDNamespace}) } return s } + +// podNamespaces are the network, IPC and UTS namespaces the sandbox holds, +// which the sandbox spec lists and containerd-sandbox-init joins. +func podNamespaces(cfg *Config, pins Pins) []specs.LinuxNamespace { + var ns []specs.LinuxNamespace + if cfg.NetNSPath != "" { + ns = append(ns, specs.LinuxNamespace{Type: specs.NetworkNamespace, Path: cfg.NetNSPath}) + } + if pins.IPC != "" { + ns = append(ns, specs.LinuxNamespace{Type: specs.IPCNamespace, Path: pins.IPC}) + } + if pins.UTS != "" { + ns = append(ns, specs.LinuxNamespace{Type: specs.UTSNamespace, Path: pins.UTS}) + } + return ns +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config_test.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config_test.go index 393f5b4204..53c25bddc7 100644 --- a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config_test.go +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config_test.go @@ -320,7 +320,7 @@ func TestBuildSpec(t *testing.T) { t.Run("pod namespaces", func(t *testing.T) { cfg := &Config{ID: testID, Hostname: "pod", NetNSPath: testNetNS, Sysctls: sysctls, Annotations: annots} - spec := buildSpec(cfg, pins, mounts) + spec := buildSpec(cfg, pins, mounts, "") assert.Equal(t, specs.Version, spec.Version) assert.Equal(t, "pod", spec.Hostname) assert.Equal(t, annots, spec.Annotations) @@ -335,15 +335,26 @@ func TestBuildSpec(t *testing.T) { }, spec.Linux.Namespaces) }) + t.Run("shared PID namespace", func(t *testing.T) { + cfg := &Config{ID: testID, Hostname: "pod", NetNSPath: testNetNS, SharedPID: true} + spec := buildSpec(cfg, pins, nil, "/proc/42/ns/pid") + assert.Equal(t, []specs.LinuxNamespace{ + {Type: specs.NetworkNamespace, Path: testNetNS}, + {Type: specs.IPCNamespace, Path: pins.IPC}, + {Type: specs.UTSNamespace, Path: pins.UTS}, + {Type: specs.PIDNamespace, Path: "/proc/42/ns/pid"}, + }, spec.Linux.Namespaces) + }) + t.Run("host network keeps only the IPC namespace", func(t *testing.T) { cfg := &Config{ID: testID, Hostname: "node", HostNetwork: true} - spec := buildSpec(cfg, Pins{IPC: pins.IPC}, nil) + spec := buildSpec(cfg, Pins{IPC: pins.IPC}, nil, "") assert.Equal(t, []specs.LinuxNamespace{{Type: specs.IPCNamespace, Path: pins.IPC}, {Type: specs.PIDNamespace}}, spec.Linux.Namespaces) }) t.Run("host ipc keeps the network and UTS namespaces", func(t *testing.T) { cfg := &Config{ID: testID, Hostname: "pod", HostIPC: true, NetNSPath: testNetNS} - spec := buildSpec(cfg, Pins{UTS: pins.UTS}, nil) + spec := buildSpec(cfg, Pins{UTS: pins.UTS}, nil, "") assert.Equal(t, []specs.LinuxNamespace{ {Type: specs.NetworkNamespace, Path: testNetNS}, {Type: specs.UTSNamespace, Path: pins.UTS}, @@ -353,7 +364,7 @@ func TestBuildSpec(t *testing.T) { t.Run("all host namespaces still carry a Linux section", func(t *testing.T) { cfg := &Config{ID: testID, Hostname: "node", HostNetwork: true, HostIPC: true, HostPID: true} - spec := buildSpec(cfg, Pins{}, nil) + spec := buildSpec(cfg, Pins{}, nil, "") require.NotNil(t, spec.Linux, "CRI tells a processless sandbox by the Linux section") assert.Empty(t, spec.Linux.Namespaces, "no PID namespace declaration for a host PID pod") }) diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/doc.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/doc.go index 780d9900d9..50621796e8 100644 --- a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/doc.go +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/doc.go @@ -46,7 +46,8 @@ // // The shim is the sandbox. CreateSandbox receives the CRI PodSandboxConfig // (as the sandbox options) and the network namespace CRI created and -// configured with CNI, and sets the pod up without a process: +// configured with CNI, and sets the pod up, without a process unless the pod +// shares its PID namespace: // // - Namespaces (namespaces.go). A locked OS thread enters the CRI network // namespace, unshares an IPC and a UTS namespace, sets the pod hostname, @@ -63,6 +64,12 @@ // host one) and mounts the pod shm tmpfs, all under its bundle instead of // the CRI state directory, and lists them as Mounts of the sandbox spec. // +// - Shared PID namespace (init.go). A pod that shares its PID namespace +// (shareProcessNamespace) needs a durable PID 1 for it, the role pause +// plays: the shim runs containerd-sandbox-init through runc, in the pod +// cgroup and the pod namespaces, and the spec names its PID namespace +// for the containers to join. Every other pod has no sandbox process. +// // - Spec (buildSpec in config.go). StartSandbox returns pid 0 and a // synthesized OCI spec that carries the namespace paths, the sysctls, the // shared file mounts and the annotations of the request. It describes what @@ -88,9 +95,7 @@ // // # Not supported yet // -// A shared pod PID namespace needs a durable PID 1 (sandbox-init); pods -// asking for one get per-container PID namespaces, with a warning (see -// buildSpec). User namespaced pods need their namespaces created inside the +// User namespaced pods need their namespaces created inside the // pod user namespace, which a multithreaded shim cannot enter; they are // rejected, as are user.* sysctls. The shim requires cgroup v2 and does not // run rootless. SELinux labels are not applied yet: the pod label allocation diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/init.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/init.go new file mode 100644 index 0000000000..5f2ad84ea9 --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/init.go @@ -0,0 +1,328 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "time" + + "github.com/containerd/errdefs" + gorunc "github.com/containerd/go-runc" + "github.com/containerd/log" + specs "github.com/opencontainers/runtime-spec/specs-go" + "golang.org/x/sys/unix" + + "github.com/containerd/containerd/v2/cmd/containerd-shim-runc-v2/process" + "github.com/containerd/containerd/v2/cmd/containerd-shim-runc-v2/runc" + "github.com/containerd/containerd/v2/contrib/seccomp" + "github.com/containerd/containerd/v2/core/containers" + "github.com/containerd/containerd/v2/pkg/namespaces" + "github.com/containerd/containerd/v2/pkg/oci" + "github.com/containerd/containerd/v2/pkg/sys/reaper" +) + +// InitBinary is the PID 1 program of a pod that shares its PID namespace, +// built from cmd/containerd-sandbox-init and installed next to the shim. +const InitBinary = "containerd-sandbox-init" + +const ( + // initPath is where the program is mounted inside its container. + initPath = "/" + InitBinary + // initRootfs is the rootfs of the init container, relative to the + // sandbox bundle: an empty tmpfs. + initRootfs = "rootfs" + // pidPin is the pin of the PID namespace the init is PID 1 of, beside + // the IPC and UTS pins. + pidPin = "pid" + // initUser is the user the init runs as: not root, and not nobody + // (65534), which workloads commonly run as. + initUser = 65535 + // initOOMScoreAdj is what CRI gives a pause container. + initOOMScoreAdj = -998 + // initStopTimeout bounds the wait for the init to be reaped after + // SIGKILL. + initStopTimeout = 10 * time.Second +) + +// initProcess is the containerd-sandbox-init runc container of a pod that +// shares its PID namespace (see the package doc). Its container id is the +// sandbox id, which is what "shim delete" force-deletes once the shim is +// gone. +type initProcess struct { + runtime *gorunc.Runc + id string + pid int + // pidNS is the pin of the PID namespace, the path the containers join. + pidNS string + // exited is closed once the exit of the process has been reaped; exit + // is valid from then on. + exited chan struct{} + exit gorunc.Exit +} + +// initBinary locates containerd-sandbox-init: next to the shim binary, then +// in PATH. +func initBinary() (string, error) { + if self, err := os.Executable(); err == nil { + p := filepath.Join(filepath.Dir(self), InitBinary) + if _, err := os.Stat(p); err == nil { + return p, nil + } + } + p, err := exec.LookPath(InitBinary) + if err != nil { + return "", fmt.Errorf("%s is not installed next to %s or in PATH: %w", InitBinary, BinaryName, errdefs.ErrFailedPrecondition) + } + return p, nil +} + +// cgroupsPath places the init container under the pod cgroup the way CRI +// places the containers of the pod: a systemd slice becomes runc's +// "slice:prefix:name", a cgroupfs path gets the container id appended. +func cgroupsPath(parent, id string) string { + if base := path.Base(parent); strings.HasSuffix(base, ".slice") { + return base + ":cri-containerd:" + id + } + return filepath.Join(parent, id) +} + +// initSpec is the OCI spec of the init container. The rootfs holds nothing +// but the program; the process has no capabilities, cannot gain privileges, +// runs under containerd's default seccomp profile and is scheduled and +// OOM-scored as a pause container is. Its /proc, that of the pod PID +// namespace, has the masked and read-only paths of a default container spec, +// as that of a pause container has. It joins the pod namespaces the sandbox +// holds and creates the PID namespace. The hostname is left alone: it is that +// of the joined UTS namespace. +// +// TODO: the pod SELinux label is not applied yet, as for the files the shim +// creates (see the SELinux TODO in CreateSandbox). +func initSpec(ctx context.Context, cfg *Config, pins Pins, binary string) (*specs.Spec, error) { + defaults, err := oci.GenerateSpec(ctx, nil, &containers.Container{ID: cfg.ID}) + if err != nil { + return nil, err + } + oomScoreAdj := initOOMScoreAdj + // The CPU shares of a pause container (WithDefaultSandboxShares). + shares := uint64(2) + s := &specs.Spec{ + Version: specs.Version, + Process: &specs.Process{ + User: specs.User{UID: initUser, GID: initUser}, + Args: []string{initPath}, + Cwd: "/", + Capabilities: &specs.LinuxCapabilities{}, + NoNewPrivileges: true, + OOMScoreAdj: &oomScoreAdj, + }, + Root: &specs.Root{Path: initRootfs, Readonly: true}, + Mounts: []specs.Mount{ + {Destination: "/proc", Type: "proc", Source: "proc", Options: []string{"nosuid", "noexec", "nodev"}}, + {Destination: "/dev", Type: "tmpfs", Source: "tmpfs", Options: []string{"nosuid", "strictatime", "mode=755", "size=65536k"}}, + {Destination: initPath, Type: "bind", Source: binary, Options: []string{"bind", "ro", "nosuid", "nodev"}}, + }, + Linux: &specs.Linux{ + Resources: &specs.LinuxResources{CPU: &specs.LinuxCPU{Shares: &shares}}, + Namespaces: append([]specs.LinuxNamespace{{Type: specs.PIDNamespace}, {Type: specs.MountNamespace}}, podNamespaces(cfg, pins)...), + MaskedPaths: defaults.Linux.MaskedPaths, + ReadonlyPaths: defaults.Linux.ReadonlyPaths, + }, + } + if cfg.CgroupParent != "" { + s.Linux.CgroupsPath = cgroupsPath(cfg.CgroupParent, cfg.ID) + } + // The profile depends on the capabilities of the process, set above. + s.Linux.Seccomp = seccomp.DefaultProfile(s) + return s, nil +} + +// createInit runs "runc create" for the init container in the sandbox +// bundle: the rootfs is a tmpfs mounted at /rootfs, so that the +// mount points runc creates in it leave nothing behind, and the spec is +// written as /config.json. From here on the process exists and is +// PID 1 of the new PID namespace, pinned at /ns/pid, waiting in runc +// for StartSandbox to exec the program. On error nothing is left running; +// the Cleanup the caller runs detaches the rootfs and the pin. +func createInit(ctx context.Context, bundle string, cfg *Config, pins Pins) (_ *initProcess, retErr error) { + ns, err := namespaces.NamespaceRequired(ctx) + if err != nil { + return nil, err + } + binary, err := initBinary() + if err != nil { + return nil, err + } + // The runc options of the runtime handler, persisted by "shim start": + // the same runc, root and cgroup driver as the containers of the pod. + opts, err := runc.ReadOptions(bundle) + if err != nil { + return nil, fmt.Errorf("failed to read the runc options: %w", err) + } + rootfs := filepath.Join(bundle, initRootfs) + if err := unix.Mount("tmpfs", rootfs, "tmpfs", unix.MS_NOSUID|unix.MS_NODEV, "mode=0755,size=1m"); err != nil { + return nil, fmt.Errorf("failed to mount the init rootfs at %s: %w", rootfs, err) + } + spec, err := initSpec(ctx, cfg, pins, binary) + if err != nil { + return nil, err + } + config, err := json.Marshal(spec) + if err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(bundle, "config.json"), config, 0o600); err != nil { + return nil, fmt.Errorf("failed to write the init spec: %w", err) + } + + p := &initProcess{ + runtime: process.NewRunc(opts.GetRoot(), bundle, ns, opts.GetBinaryName(), opts.GetSystemdCgroup()), + id: cfg.ID, + exited: make(chan struct{}), + } + // Subscribed before the process exists so that its exit cannot be + // missed; the watcher learns the pid once runc reports it. + ec := reaper.Default.Subscribe() + pidCh := make(chan int, 1) + go p.watch(ec, pidCh) + defer func() { + if retErr != nil { + close(pidCh) + // A failed create may have left runc state and a process. + if err := p.runtime.Delete(ctx, p.id, &gorunc.DeleteOpts{Force: true}); err != nil { + log.G(ctx).WithError(err).WithField("id", p.id).Debugf("failed to delete %s after a failed create", InitBinary) + } + } + }() + + nullIO, err := gorunc.NewNullIO() + if err != nil { + return nil, err + } + defer nullIO.Close() + pidFile := filepath.Join(bundle, process.InitPidFile) + if err := p.runtime.Create(ctx, p.id, bundle, &gorunc.CreateOpts{ + IO: nullIO, + PidFile: pidFile, + NoPivot: opts.GetNoPivotRoot(), + NoNewKeyring: opts.GetNoNewKeyring(), + }); err != nil { + return nil, fmt.Errorf("failed to create the %s container: %w", InitBinary, err) + } + if p.pid, err = gorunc.ReadPidFile(pidFile); err != nil { + return nil, fmt.Errorf("failed to read the %s pid: %w", InitBinary, err) + } + pidCh <- p.pid + // Pinned like the IPC and UTS namespaces, so that the path the + // containers join never names whatever process reuses the pid. + if err := os.MkdirAll(filepath.Join(bundle, pinDir), 0o700); err != nil { + return nil, fmt.Errorf("failed to create namespace pin directory: %w", err) + } + p.pidNS = filepath.Join(bundle, pinDir, pidPin) + if err := pin(fmt.Sprintf("/proc/%d/ns/pid", p.pid), p.pidNS); err != nil { + return nil, err + } + return p, nil +} + +// watch delivers the exit of the init through exited. It drains the reaper +// subscription throughout (a subscriber that falls behind slows every reap +// in the shim) and remembers the exits seen before the pid is known, since +// the process can be reaped between "runc create" and the pid file read. +func (p *initProcess) watch(ec chan gorunc.Exit, pidCh <-chan int) { + defer reaper.Default.Unsubscribe(ec) + pid := -1 + seen := map[int]gorunc.Exit{} + for { + select { + case id, ok := <-pidCh: + if !ok { + return + } + if e, ok := seen[id]; ok { + p.finish(e) + return + } + pid, seen = id, nil + case e, ok := <-ec: + if !ok { + return + } + switch { + case pid < 0: + seen[e.Pid] = e + case e.Pid == pid: + p.finish(e) + return + } + } + } +} + +func (p *initProcess) finish(e gorunc.Exit) { + p.exit = e + close(p.exited) +} + +// stop ends the init with SIGKILL, the one signal it does not ignore, waits +// for its exit to be reaped and deletes its runc container. A process that +// already exited on its own is only deleted; the exit is the same either +// way. The wait is bounded by initStopTimeout. +func (p *initProcess) stop(ctx context.Context) (gorunc.Exit, error) { + exit, err := p.kill(ctx) + if err != nil { + return exit, err + } + return exit, p.delete(ctx) +} + +// kill ends the init with SIGKILL, unless it has exited already, and waits +// for its exit, at most initStopTimeout. +func (p *initProcess) kill(ctx context.Context) (gorunc.Exit, error) { + select { + case <-p.exited: + default: + if err := p.runtime.Kill(ctx, p.id, int(unix.SIGKILL), nil); err != nil { + // The process may have exited in the meantime; the wait tells. + log.G(ctx).WithError(err).WithField("id", p.id).Debugf("failed to kill %s", InitBinary) + } + select { + case <-p.exited: + case <-time.After(initStopTimeout): + return gorunc.Exit{}, fmt.Errorf("%s (pid %d) did not exit after SIGKILL", InitBinary, p.pid) + case <-ctx.Done(): + return gorunc.Exit{}, ctx.Err() + } + } + return p.exit, nil +} + +// delete removes the runc container of the init. Deleting it again is fine. +func (p *initProcess) delete(ctx context.Context) error { + if err := p.runtime.Delete(ctx, p.id, &gorunc.DeleteOpts{Force: true}); err != nil { + return fmt.Errorf("failed to delete the %s container: %w", InitBinary, err) + } + return nil +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/init_test.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/init_test.go new file mode 100644 index 0000000000..1e314bfdef --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/init_test.go @@ -0,0 +1,202 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "fmt" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "testing" + "time" + + gorunc "github.com/containerd/go-runc" + "github.com/moby/sys/mountinfo" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" + + "github.com/containerd/containerd/v2/contrib/seccomp" + "github.com/containerd/containerd/v2/core/containers" + "github.com/containerd/containerd/v2/pkg/namespaces" + "github.com/containerd/containerd/v2/pkg/oci" + "github.com/containerd/containerd/v2/pkg/sys/reaper" + "github.com/containerd/containerd/v2/pkg/testutil" +) + +func TestCgroupsPath(t *testing.T) { + assert.Equal(t, "/kubepods/burstable/pod1/"+testID, cgroupsPath("/kubepods/burstable/pod1", testID)) + assert.Equal(t, "kubepods-burstable-pod1.slice:cri-containerd:"+testID, cgroupsPath("/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod1.slice", testID)) + assert.Equal(t, "test.slice:cri-containerd:"+testID, cgroupsPath("/test.slice", testID)) +} + +func TestInitSpec(t *testing.T) { + ctx := namespaces.WithNamespace(context.Background(), "testing") + cfg := &Config{ID: testID, NetNSPath: testNetNS, CgroupParent: "/kubepods/pod1", SharedPID: true} + pins := Pins{IPC: "/bundle/ns/ipc", UTS: "/bundle/ns/uts"} + binary := "/usr/local/bin/" + InitBinary + spec, err := initSpec(ctx, cfg, pins, binary) + require.NoError(t, err) + + require.NotNil(t, spec.Process) + assert.Equal(t, []string{"/" + InitBinary}, spec.Process.Args) + assert.Equal(t, specs.User{UID: initUser, GID: initUser}, spec.Process.User) + assert.True(t, spec.Process.NoNewPrivileges) + assert.Equal(t, &specs.LinuxCapabilities{}, spec.Process.Capabilities, "no capabilities") + require.NotNil(t, spec.Process.OOMScoreAdj) + assert.Equal(t, initOOMScoreAdj, *spec.Process.OOMScoreAdj) + assert.Equal(t, &specs.Root{Path: "rootfs", Readonly: true}, spec.Root) + assert.Equal(t, specs.Mount{Destination: "/" + InitBinary, Type: "bind", Source: binary, Options: []string{"bind", "ro", "nosuid", "nodev"}}, spec.Mounts[len(spec.Mounts)-1]) + assert.Equal(t, []specs.LinuxNamespace{ + {Type: specs.PIDNamespace}, + {Type: specs.MountNamespace}, + {Type: specs.NetworkNamespace, Path: testNetNS}, + {Type: specs.IPCNamespace, Path: pins.IPC}, + {Type: specs.UTSNamespace, Path: pins.UTS}, + }, spec.Linux.Namespaces) + assert.Equal(t, "/kubepods/pod1/"+testID, spec.Linux.CgroupsPath) + assert.Empty(t, spec.Hostname, "the hostname of the joined UTS namespace is left alone") + + defaults, err := oci.GenerateSpec(ctx, nil, &containers.Container{ID: testID}) + require.NoError(t, err) + assert.Equal(t, defaults.Linux.MaskedPaths, spec.Linux.MaskedPaths) + assert.Equal(t, defaults.Linux.ReadonlyPaths, spec.Linux.ReadonlyPaths) + assert.Contains(t, spec.Linux.ReadonlyPaths, "/proc/sys") + assert.Equal(t, seccomp.DefaultProfile(spec), spec.Linux.Seccomp) + + t.Run("host namespaces", func(t *testing.T) { + spec, err := initSpec(ctx, &Config{ID: testID, HostNetwork: true, HostIPC: true, SharedPID: true}, Pins{}, binary) + require.NoError(t, err) + assert.Equal(t, []specs.LinuxNamespace{{Type: specs.PIDNamespace}, {Type: specs.MountNamespace}}, spec.Linux.Namespaces) + assert.Empty(t, spec.Linux.CgroupsPath) + }) +} + +// startReaper makes the test process reap its children the way the shim +// does; the runc commands and the exit tracking of the init rely on it. +func startReaper(t *testing.T) { + require.NoError(t, reaper.SetSubreaper(1)) + monitor := gorunc.Monitor + gorunc.Monitor = reaper.Default + ch := make(chan os.Signal, 32) + signal.Notify(ch, unix.SIGCHLD) + done := make(chan struct{}) + go func() { + for { + select { + case <-ch: + _ = reaper.Reap() + case <-done: + return + } + } + }() + t.Cleanup(func() { + signal.Stop(ch) + close(done) + gorunc.Monitor = monitor + _ = reaper.SetSubreaper(0) + }) +} + +// newInit creates the init of a pod in the host network and IPC namespaces +// in a fresh sandbox bundle, stopped and cleaned up with the test. The test +// needs root, runc and the installed init binary. +func newInit(t *testing.T) (context.Context, string, *initProcess) { + testutil.RequiresRoot(t) + if _, err := exec.LookPath("runc"); err != nil { + t.Skip("runc is not installed") + } + if _, err := initBinary(); err != nil { + t.Skip(err.Error()) + } + startReaper(t) + + ctx := namespaces.WithNamespace(context.Background(), "testing") + bundle := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(bundle, initRootfs), 0o711)) + t.Cleanup(func() { _ = Cleanup(ctx, bundle) }) + cfg := &Config{ID: fmt.Sprintf("sandbox-init-test-%d", os.Getpid()), HostNetwork: true, HostIPC: true, SharedPID: true, CgroupParent: "/containerd-test"} + p, err := createInit(ctx, bundle, cfg, Pins{}) + require.NoError(t, err) + t.Cleanup(func() { _, _ = p.stop(ctx) }) + return ctx, bundle, p +} + +// startInit execs the program of p and waits for it to run. +func startInit(ctx context.Context, t *testing.T, p *initProcess) { + require.NoError(t, p.runtime.Start(ctx, p.id)) + require.Eventually(t, func() bool { return running(p.pid) }, 5*time.Second, 20*time.Millisecond) +} + +// running reports whether the init program is running as pid: the comm of +// the process is the binary name, truncated by the kernel. +func running(pid int) bool { + comm, err := os.ReadFile(fmt.Sprintf("/proc/%d/comm", pid)) + return err == nil && strings.HasPrefix(InitBinary, strings.TrimSpace(string(comm))) +} + +func TestInitLifecycle(t *testing.T) { + ctx, bundle, p := newInit(t) + assert.Greater(t, p.pid, 0) + mounted, err := mountinfo.Mounted(p.pidNS) + require.NoError(t, err) + assert.True(t, mounted, "the PID namespace is pinned") + assert.False(t, running(p.pid), "created, not started") + + startInit(ctx, t, p) + + exit, err := p.stop(ctx) + require.NoError(t, err) + assert.Equal(t, 128+int(unix.SIGKILL), exit.Status) + assert.NoFileExists(t, fmt.Sprintf("/proc/%d", p.pid)) + + // Stopping again deletes what is already deleted and reports the same exit. + again, err := p.stop(ctx) + require.NoError(t, err) + assert.Equal(t, exit, again) + + require.NoError(t, Cleanup(ctx, bundle)) + for _, p := range []string{filepath.Join(bundle, initRootfs), p.pidNS} { + mounted, err := mountinfo.Mounted(p) + if err == nil { + assert.False(t, mounted, "%s is detached", p) + } + } +} + +func TestInitExit(t *testing.T) { + ctx, _, p := newInit(t) + startInit(ctx, t, p) + + // An init killed from outside the shim is noticed and its exit kept. + require.NoError(t, unix.Kill(p.pid, unix.SIGKILL)) + select { + case <-p.exited: + case <-time.After(5 * time.Second): + t.Fatal("the exit of the init was not seen") + } + exit, err := p.stop(ctx) + require.NoError(t, err) + assert.Equal(t, 128+int(unix.SIGKILL), exit.Status) +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service.go index e08b3afeae..f2d5ca2cca 100644 --- a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service.go +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service.go @@ -21,6 +21,7 @@ package sandbox import ( "context" "fmt" + "strconv" "sync" "time" @@ -82,12 +83,18 @@ type instance struct { pins Pins spec *anypb.Any createdAt time.Time + // init is PID 1 of the pod PID namespace, nil unless the pod shares it. + init *initProcess - started bool - stopped bool - exitedAt time.Time + started bool + stopped bool + exitStatus uint32 + exitedAt time.Time // done is closed once the sandbox is stopped. done chan struct{} + // released is set once the init container, the pins and the files are + // gone; a stopped sandbox may still hold them if releasing failed. + released bool } var _ api.TTRPCSandboxService = (*Service)(nil) @@ -133,9 +140,6 @@ func (s *Service) CreateSandbox(ctx context.Context, r *api.CreateSandboxRequest return nil, errgrpc.ToGRPC(err) } log.G(ctx).WithField("id", cfg.ID).Warnf("%s is experimental: the pod sandbox runs without a pause container", BinaryName) - if cfg.SharedPID { - log.G(ctx).WithField("id", cfg.ID).Warnf("%s cannot share a pod PID namespace yet: the containers of this pod get their own PID namespaces", BinaryName) - } defer func() { if retErr != nil { @@ -161,25 +165,70 @@ func (s *Service) CreateSandbox(ctx context.Context, r *api.CreateSandboxRequest if err != nil { return nil, errgrpc.ToGRPCf(err, "failed to set up the pod files") } - spec, err := typeurl.MarshalAnyToProto(buildSpec(cfg, pins, mounts)) + var ( + initP *initProcess + pidNS string + ) + if cfg.SharedPID { + // Created last, once the pins it joins exist; if anything below + // fails it is stopped before the deferred Cleanup runs. + if initP, err = createInit(ctx, bundle, cfg, pins); err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to create the pod PID namespace") + } + defer func() { + if retErr != nil { + if _, err := initP.stop(ctx); err != nil { + log.G(ctx).WithError(err).WithField("id", cfg.ID).Warnf("failed to stop %s after a failed sandbox creation", InitBinary) + } + } + }() + pidNS = initP.pidNS + } + spec, err := typeurl.MarshalAnyToProto(buildSpec(cfg, pins, mounts, pidNS)) if err != nil { return nil, errgrpc.ToGRPCf(err, "failed to marshal the sandbox spec") } - s.sandbox = &instance{ + inst := &instance{ cfg: cfg, bundle: bundle, pins: pins, spec: spec, createdAt: time.Now(), + init: initP, done: make(chan struct{}), } + s.sandbox = inst + if initP != nil { + // The watcher outlives this request: it keeps the namespace and + // log fields of the request without its cancellation. + go s.watchInit(context.WithoutCancel(ctx), inst) + } return &api.CreateSandboxResponse{}, nil } -// StartSandbox marks the sandbox running. There is no process to start: the -// pid is 0 and the spec describes what the sandbox holds. Starting it again -// returns the same response. +// watchInit ends the sandbox when its init exits on its own: the kernel +// kills every process in a PID namespace whose PID 1 is gone, so the pod is +// over, as it is when a pause container dies. StopSandbox, which kills the +// init itself, records the exit first and leaves nothing to do here. +func (s *Service) watchInit(ctx context.Context, inst *instance) { + <-inst.init.exited + s.mu.Lock() + defer s.mu.Unlock() + if inst.stopped { + return + } + log.G(ctx).WithField("id", inst.cfg.ID).Warnf("%s exited with status %d, stopping the sandbox", InitBinary, inst.init.exit.Status) + if err := s.stop(ctx, inst); err != nil { + log.G(ctx).WithError(err).WithField("id", inst.cfg.ID).Error("failed to stop the sandbox after its init exited") + } +} + +// StartSandbox marks the sandbox running. There is no process to start +// unless the pod shares its PID namespace, in which case the program of its +// PID 1 is exec'd now. The pid reported is 0 either way: the sandbox is +// path-backed and the spec describes what it holds, that PID namespace +// included. Starting it again returns the same response. func (s *Service) StartSandbox(ctx context.Context, r *api.StartSandboxRequest) (*api.StartSandboxResponse, error) { s.mu.Lock() defer s.mu.Unlock() @@ -190,6 +239,11 @@ func (s *Service) StartSandbox(ctx context.Context, r *api.StartSandboxRequest) if inst.stopped { return nil, errgrpc.ToGRPCf(errdefs.ErrFailedPrecondition, "sandbox %q is stopped", inst.cfg.ID) } + if !inst.started && inst.init != nil { + if err := inst.init.runtime.Start(ctx, inst.init.id); err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to start %s", InitBinary) + } + } inst.started = true return &api.StartSandboxResponse{ Pid: 0, @@ -205,9 +259,10 @@ func (s *Service) Platform(ctx context.Context, r *api.PlatformRequest) (*api.Pl }, nil } -// StopSandbox releases the pod namespaces and files. Containers still running -// keep the namespaces they joined until they exit; CRI stops them first. -// Stopping twice is fine. +// StopSandbox releases the pod namespaces and files, and kills the init of a +// shared-PID pod, which ends whatever is left in the pod PID namespace. +// Containers still running keep the other namespaces they joined until they +// exit; CRI stops them first. Stopping twice is fine. func (s *Service) StopSandbox(ctx context.Context, r *api.StopSandboxRequest) (*api.StopSandboxResponse, error) { s.mu.Lock() defer s.mu.Unlock() @@ -221,22 +276,42 @@ func (s *Service) StopSandbox(ctx context.Context, r *api.StopSandboxRequest) (* return &api.StopSandboxResponse{}, nil } -// stop releases what the sandbox holds and marks it stopped. The caller holds -// s.mu. +// stop marks the sandbox stopped and releases what it holds. The init of a +// shared-PID pod is killed first, and its exit becomes the exit of the +// sandbox. Once the init is gone the sandbox is stopped, and Status and Wait +// report it, even if the release then fails: the pod PID namespace died with +// its init. A release that failed is retried by the next stop. The caller +// holds s.mu. func (s *Service) stop(ctx context.Context, inst *instance) error { - if inst.stopped { + if !inst.stopped { + exitedAt := time.Now() + if inst.init != nil { + exit, err := inst.init.kill(ctx) + if err != nil { + return fmt.Errorf("failed to stop sandbox %q: %w", inst.cfg.ID, err) + } + inst.exitStatus, exitedAt = uint32(exit.Status), exit.Timestamp + } + inst.stopped, inst.exitedAt = true, exitedAt + close(inst.done) + } + if inst.released { return nil } + if inst.init != nil { + if err := inst.init.delete(ctx); err != nil { + return fmt.Errorf("failed to stop sandbox %q: %w", inst.cfg.ID, err) + } + } if err := Cleanup(ctx, inst.bundle); err != nil { return fmt.Errorf("failed to clean up sandbox %q: %w", inst.cfg.ID, err) } - inst.stopped = true - inst.exitedAt = time.Now() - close(inst.done) + inst.released = true return nil } -// WaitSandbox blocks until the sandbox is stopped. +// WaitSandbox blocks until the sandbox is stopped. The exit status is that +// of the init of a shared-PID pod, 0 for a sandbox without a process. func (s *Service) WaitSandbox(ctx context.Context, r *api.WaitSandboxRequest) (*api.WaitSandboxResponse, error) { s.mu.Lock() inst, err := s.findSandbox(r.GetSandboxID()) @@ -250,7 +325,7 @@ func (s *Service) WaitSandbox(ctx context.Context, r *api.WaitSandboxRequest) (* return nil, errgrpc.ToGRPC(ctx.Err()) } return &api.WaitSandboxResponse{ - ExitStatus: 0, + ExitStatus: inst.exitStatus, ExitedAt: protobuf.ToTimestamp(inst.exitedAt), }, nil } @@ -278,12 +353,17 @@ func (s *Service) SandboxStatus(ctx context.Context, r *api.SandboxStatusRequest resp.State = runtime.PodSandboxState_SANDBOX_READY.String() } if r.GetVerbose() { - resp.Info = make(map[string]string, 4) + var initPid string + if inst.init != nil { + initPid = strconv.Itoa(inst.init.pid) + } + resp.Info = make(map[string]string, 5) for _, kv := range [][2]string{ {BinaryName + "/netns", inst.cfg.NetNSPath}, {BinaryName + "/ipcns", inst.pins.IPC}, {BinaryName + "/utsns", inst.pins.UTS}, {BinaryName + "/hostname", inst.cfg.Hostname}, + {BinaryName + "/init-pid", initPid}, } { if kv[1] != "" { resp.Info[kv[0]] = kv[1] @@ -327,7 +407,8 @@ func (s *Service) ShutdownSandbox(ctx context.Context, r *api.ShutdownSandboxReq // SandboxMetrics reports the cgroup v2 stats of the pod cgroup, encoded the // way CRI expects sandbox metrics (a *cgroup2/stats.Metrics). The sandbox has -// no process of its own, so the aggregate pod cgroup is the only meaningful +// no process of its own (the init of a shared-PID pod sits in the pod cgroup +// like any container), so the aggregate pod cgroup is the only meaningful // source. CRI polls this every second; the cgroup files are read outside the // lock. func (s *Service) SandboxMetrics(ctx context.Context, r *api.SandboxMetricsRequest) (*api.SandboxMetricsResponse, error) { diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service_test.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service_test.go index 392a4b8c49..855c36ddd2 100644 --- a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service_test.go +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service_test.go @@ -20,6 +20,8 @@ package sandbox import ( "context" + "os" + "path/filepath" "testing" "time" @@ -74,6 +76,33 @@ func TestServiceWithoutSandbox(t *testing.T) { } } +// TestStopRetriesRelease checks that a sandbox is stopped, and Wait returns, +// even when releasing what it holds fails, and that the next stop retries the +// release. +func TestStopRetriesRelease(t *testing.T) { + ctx := context.Background() + bundle := t.TempDir() + // A file where the pin directory belongs makes Cleanup fail. + pins := filepath.Join(bundle, pinDir) + require.NoError(t, os.WriteFile(pins, nil, 0o600)) + inst := &instance{cfg: &Config{ID: testID}, bundle: bundle, done: make(chan struct{})} + s := &Service{} + + require.Error(t, s.stop(ctx, inst)) + assert.True(t, inst.stopped) + assert.False(t, inst.released) + select { + case <-inst.done: + default: + t.Fatal("a stopped sandbox must end Wait even if its release failed") + } + + require.NoError(t, os.Remove(pins)) + require.NoError(t, s.stop(ctx, inst)) + assert.True(t, inst.released) + require.NoError(t, s.stop(ctx, inst), "stopping a released sandbox is a no-op") +} + func TestSandboxStatusBeforeStart(t *testing.T) { ctx := context.Background() _, sd := shutdown.WithShutdown(ctx) From 7b8fd44aefe0e6d03bba7ceae084c811e70ab112 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Fri, 23 Oct 2026 13:24:50 -0700 Subject: [PATCH 12/13] ci: run the PodPID critest spec through the sandboxed shim The sandboxed shim shares a pod PID namespace now, so the critest step for it no longer skips "runtime should support PodPID". The user namespace specs still skip themselves. Signed-off-by: Maksym Pavlenko --- .github/workflows/ci.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 084560e669..808cf50273 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -609,14 +609,10 @@ jobs: # Experimental: the same critest suite through the pause-less sandbox # shim (containerd-shim-sandboxed-runc-v2 selected with runtime_path, - # with the shim sandboxer). Specs that need what the shim does not - # support yet are skipped by their full name; the list shrinks as the - # features land: - # - "runtime should support PodPID": a shared pod PID namespace needs - # a durable PID 1 (sandbox-init). - # The user namespace specs skip themselves: the handler does not - # advertise user namespaces until the shim supports them. - # The sandboxed shim is a Go shim: it runs on the runc.v2 legs only. + # with the shim sandboxer). The user namespace specs skip themselves: + # the handler does not advertise user namespaces until the shim + # supports them. The sandboxed shim is a Go shim: it runs on the + # runc.v2 legs only. - if: matrix.runtime == 'io.containerd.runc.v2' name: cri-tools critest (sandbox shim) env: @@ -624,7 +620,6 @@ jobs: TEST_RUNTIME_PATH: /usr/local/bin/containerd-shim-sandboxed-runc-v2 SANDBOXER: shim CGROUP_DRIVER: ${{ matrix.cgroup_driver }} - SKIP_TEST: "NamespaceOption runtime should support PodPID" run: | env sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report-sandbox-shim" From fe4cdb0bf88434b97757b66aabca42f9c359d662 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Wed, 4 Nov 2026 18:26:45 -0800 Subject: [PATCH 13/13] sandbox: clean up a failed shim with a context of its own When CreateSandbox or StartSandbox fails because the caller's context was canceled or ran past its deadline, cleanupShim passed that same context to ShutdownSandbox and derived the shim deletion timeout from it, so both failed at once and the shim, its sandbox and the mounts under its bundle were left behind. The cleanup now keeps the values of the caller's context but not its cancellation, and bounds the shutdown and the deletion with timeouts of their own; the shutdown one leaves the shim time to stop what it runs for the sandbox. A closed connection from a shim that exited before answering ShutdownSandbox is not logged as a failure, as Shutdown already treats it as success. Signed-off-by: Maksym Pavlenko --- plugins/sandbox/controller.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/plugins/sandbox/controller.go b/plugins/sandbox/controller.go index 1d237a2560..5402afe2a1 100644 --- a/plugins/sandbox/controller.go +++ b/plugins/sandbox/controller.go @@ -104,10 +104,22 @@ type controllerLocal struct { var _ sandbox.Controller = (*controllerLocal)(nil) func (c *controllerLocal) cleanupShim(ctx context.Context, sandboxID string, svc runtimeAPI.TTRPCSandboxService) { - // Let the shim exit, then we can clean up the bundle after. - if _, sErr := svc.ShutdownSandbox(ctx, &runtimeAPI.ShutdownSandboxRequest{ + // The caller's context may be the reason Create or Start failed, canceled + // or past its deadline; the cleanup must still run, so it keeps the values + // of that context (namespace, logger) but not its cancellation, and each + // step gets its own deadline instead. + ctx = context.WithoutCancel(ctx) + + // Let the shim exit, then we can clean up the bundle after. The deadline + // leaves room for the shim to stop what it runs for the sandbox, which + // takes its own bounded wait. A closed connection means the shim exited + // before answering, as for Shutdown. + sctx, scancel := context.WithTimeout(ctx, 30*time.Second) + _, sErr := svc.ShutdownSandbox(sctx, &runtimeAPI.ShutdownSandboxRequest{ SandboxID: sandboxID, - }); sErr != nil { + }) + scancel() + if sErr != nil && !errors.Is(sErr, ttrpc.ErrClosed) { log.G(ctx).WithError(sErr).WithField("sandboxID", sandboxID). Error("failed to shutdown sandbox") }