diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c73f51d39..808cf50273 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -607,6 +607,23 @@ jobs: env sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report" + # Experimental: the same critest suite through the pause-less sandbox + # shim (containerd-shim-sandboxed-runc-v2 selected with runtime_path, + # with the shim sandboxer). The user namespace specs skip themselves: + # the handler does not advertise user namespaces until the shim + # supports them. The sandboxed shim is a Go shim: it runs on the + # runc.v2 legs only. + - if: matrix.runtime == 'io.containerd.runc.v2' + name: cri-tools critest (sandbox shim) + env: + TEST_RUNTIME: ${{ matrix.runtime }} + TEST_RUNTIME_PATH: /usr/local/bin/containerd-shim-sandboxed-runc-v2 + SANDBOXER: shim + CGROUP_DRIVER: ${{ matrix.cgroup_driver }} + run: | + env + sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report-sandbox-shim" + # The Rust shim does not implement checkpoint yet. - if: matrix.os != 'ubuntu-24.04-arm' && matrix.runtime != 'io.containerd.runc.v2-rs' name: Checkpoint/Restore Disable via CRI @@ -641,6 +658,8 @@ jobs: *-gotest.json ${{github.workspace}}/report/*.xml ${{github.workspace}}/report/*.log + ${{github.workspace}}/report-sandbox-shim/*.xml + ${{github.workspace}}/report-sandbox-shim/*.log integration-vm: name: VM integration diff --git a/Makefile b/Makefile index a39c97e0f8..6b7c423d29 100644 --- a/Makefile +++ b/Makefile @@ -272,6 +272,14 @@ bin/containerd-shim-runc-v2: cmd/containerd-shim-runc-v2 FORCE # set !cgo and om @echo "$(WHALE) $@" CGO_ENABLED=${SHIM_CGO_ENABLED} $(GO) build ${GO_BUILD_FLAGS} -o $@ ${SHIM_GO_LDFLAGS} ${SHIM_GO_TAGS} ./cmd/containerd-shim-runc-v2 +bin/containerd-shim-sandboxed-runc-v2: cmd/containerd-shim-sandboxed-runc-v2 FORCE # experimental pause-less sandbox shim, static like containerd-shim-runc-v2 + @echo "$(WHALE) $@" + CGO_ENABLED=${SHIM_CGO_ENABLED} $(GO) build ${GO_BUILD_FLAGS} -o $@ ${SHIM_GO_LDFLAGS} ${SHIM_GO_TAGS} ./cmd/containerd-shim-sandboxed-runc-v2 + +bin/containerd-sandbox-init: cmd/containerd-sandbox-init FORCE # PID 1 of a shared pod PID namespace for the sandboxed shim, static like the shims + @echo "$(WHALE) $@" + CGO_ENABLED=${SHIM_CGO_ENABLED} $(GO) build ${GO_BUILD_FLAGS} -o $@ ${SHIM_GO_LDFLAGS} ${SHIM_GO_TAGS} ./cmd/containerd-sandbox-init + binaries: $(BINARIES) ## build binaries @echo "$(WHALE) $@" diff --git a/Makefile.linux b/Makefile.linux index 6f44a29336..6f50624298 100644 --- a/Makefile.linux +++ b/Makefile.linux @@ -17,6 +17,8 @@ WHALE="+" ONI="-" COMMANDS += containerd-shim-runc-v2 +COMMANDS += containerd-shim-sandboxed-runc-v2 +COMMANDS += containerd-sandbox-init # check GOOS for cross compile builds ifeq ($(GOOS),linux) diff --git a/cmd/containerd-sandbox-init/main.go b/cmd/containerd-sandbox-init/main.go new file mode 100644 index 0000000000..4b2a951c58 --- /dev/null +++ b/cmd/containerd-sandbox-init/main.go @@ -0,0 +1,134 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// containerd-sandbox-init is PID 1 of the PID namespace of a pod that shares +// it (shareProcessNamespace). containerd-shim-sandboxed-runc-v2, the +// pause-less sandbox shim, starts it through runc for those pods only, and it +// runs as a child of the shim. An ordinary pod, one that does not share its +// PID namespace, has no sandbox process at all. +// +// It does what PID 1 of a pod has to do and nothing else: it stays alive so +// that the namespace does, it lets the kernel reap the orphans that get +// reparented to it, and it cannot be ended from inside the pod. +// +// The kernel delivers a signal to the init of a PID namespace, from inside +// that namespace, only when init has a handler installed for it, and discards +// it otherwise. SIGKILL and SIGSTOP reach it from the parent namespace only, +// and SIGKILL from the shim is how it ends. The Go runtime installs handlers +// for almost every signal and ends the process on most of them. A Go init +// that kept those handlers could be killed from a member container with +// kill -HUP 1 or kill -SEGV 1, so disableSignals removes every one of them. +// +// It is also not dumpable, so that the other processes of the pod cannot +// inspect or control it: without CAP_SYS_PTRACE they can neither ptrace it +// nor open /proc/1/root, /proc/1/mem and the like. +package main + +import ( + "os" + "os/signal" + "runtime" + "runtime/debug" + "syscall" + "unsafe" + + "golang.org/x/sys/unix" +) + +func main() { + run() +} + +// run sets the process up and parks it; it never returns. +func run() { + // runc makes the process non-dumpable, but execve made it dumpable + // again, since it changed no credentials. + if err := unix.Prctl(unix.PR_SET_DUMPABLE, 0, 0, 0, 0); err != nil { + os.Exit(1) + } + // A single scheduler thread (GOMAXPROCS) and no garbage collector: + // there is nothing to run and nothing to collect. Like pause, the + // process then sleeps until it is killed. + runtime.GOMAXPROCS(1) + debug.SetGCPercent(-1) + if err := disableSignals(); err != nil { + os.Exit(1) + } + debug.FreeOSMemory() + for { + // A blocking system call rather than select {}: with no other + // goroutine the runtime would report a deadlock. + unix.Pause() + } +} + +// disableSignals leaves no signal handler in place, so that no signal sent +// from inside the pod runs any code in the process. SIGCHLD is ignored rather +// than defaulted so that the kernel reaps the children reparented to PID 1 on +// its own, with no wait loop. Ignoring SIGURG gives up asynchronous +// preemption, which a process whose only goroutine waits in pause(2) does not +// need. os/signal cannot touch the synchronous faults, for which the runtime +// installs handlers that exit the process when the signal came from kill(2), +// nor SIGPROF, whose handler it keeps for profiling, nor signals 32 and 33, +// which the C library reserves and the runtime keeps for itself (33 for +// system calls that must run on every thread, which this process never +// makes): the faults go back to the kernel default and the others are +// ignored, through rt_sigaction. +func disableSignals() error { + // Signals 32 and 33, reserved by the C library and kept by the Go runtime. + const ( + sigRuntime1 = syscall.Signal(32) + sigRuntime2 = syscall.Signal(33) + ) + for n := 1; n <= 64; n++ { + sig := syscall.Signal(n) + switch sig { + case unix.SIGKILL, unix.SIGSTOP: + // Uncatchable, and how the shim ends the process. + case unix.SIGILL, unix.SIGTRAP, unix.SIGBUS, unix.SIGFPE, unix.SIGSEGV, unix.SIGSTKFLT, unix.SIGSYS: + if err := setAction(sig, sigDFL); err != nil { + return err + } + case unix.SIGPROF, sigRuntime1, sigRuntime2: + if err := setAction(sig, sigIGN); err != nil { + return err + } + default: + signal.Ignore(sig) + } + } + return nil +} + +// The dispositions setAction sets. +const ( + sigDFL = 0 + sigIGN = 1 +) + +// setAction sets the disposition of sig, with no flags and an empty mask. The +// struct sigaction passed is sized for every architecture and starts with the +// handler, as it does on every supported one (MIPS puts the flags first and +// has a larger sigset); the last argument is the size of the kernel sigset. +func setAction(sig syscall.Signal, handler uintptr) error { + act := [4]uint64{uint64(handler)} + if _, _, errno := unix.RawSyscall6(unix.SYS_RT_SIGACTION, uintptr(sig), uintptr(unsafe.Pointer(&act)), 0, 8, 0, 0); errno != 0 { + return errno + } + return nil +} diff --git a/cmd/containerd-sandbox-init/main_test.go b/cmd/containerd-sandbox-init/main_test.go new file mode 100644 index 0000000000..dc0a656583 --- /dev/null +++ b/cmd/containerd-sandbox-init/main_test.go @@ -0,0 +1,164 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" + + "github.com/containerd/containerd/v2/pkg/testutil" +) + +// childEnv makes the test binary run as the init instead of the tests. +const childEnv = "CONTAINERD_SANDBOX_INIT_TEST_CHILD" + +func TestMain(m *testing.M) { + if os.Getenv(childEnv) == "1" { + run() + } + os.Exit(m.Run()) +} + +// initUser is the user the shim runs the init as. +const initUser = 65535 + +// TestSignals runs the init as PID 1 of a new PID namespace, as the user the +// shim runs it as, and sends it every signal but SIGKILL and SIGSTOP, from +// the parent namespace: a signal that ended it here would end it from inside +// the pod as well (see the package doc). +func TestSignals(t *testing.T) { + testutil.RequiresRoot(t) + + cmd := exec.Command(initCopy(t)) + cmd.Env = append(os.Environ(), childEnv+"=1") + cmd.SysProcAttr = &syscall.SysProcAttr{ + Cloneflags: unix.CLONE_NEWPID, + Credential: &syscall.Credential{Uid: initUser, Gid: initUser}, + } + require.NoError(t, cmd.Start()) + // waitErr is valid once exited is closed. + var waitErr error + exited := make(chan struct{}) + go func() { + waitErr = cmd.Wait() + close(exited) + }() + t.Cleanup(func() { + _ = cmd.Process.Kill() + <-exited + }) + pid := cmd.Process.Pid + + // The setup is done once the last signal of the loop, 64, is ignored. + require.Eventually(t, func() bool { + ignored, _ := signalMasks(t, pid) + return ignored&(1<<63) != 0 + }, 10*time.Second, 10*time.Millisecond, "the init did not finish its signal setup") + + ignored, caught := signalMasks(t, pid) + for _, sig := range []syscall.Signal{unix.SIGCHLD, unix.SIGURG, unix.SIGPROF} { + assert.NotZero(t, ignored&(1<<(sig-1)), "%s must be ignored", unix.SignalName(sig)) + } + assert.Zero(t, caught, "no signal may have a handler (SigCgt %#x)", caught) + + // The /proc/ entries of a process that is not dumpable belong to + // root (the directory itself keeps the owner of the process). + var st unix.Stat_t + require.NoError(t, unix.Stat(fmt.Sprintf("/proc/%d/status", pid), &st)) + assert.EqualValues(t, 0, st.Uid, "the init must not be dumpable") + + for n := 1; n <= 64; n++ { + sig := syscall.Signal(n) + if sig == unix.SIGKILL || sig == unix.SIGSTOP { + continue + } + require.NoError(t, unix.Kill(pid, sig), "signal %d", n) + select { + case <-exited: + t.Fatalf("signal %d (%s) ended the init: %v", n, unix.SignalName(sig), waitErr) + case <-time.After(20 * time.Millisecond): + } + } + + require.NoError(t, unix.Kill(pid, unix.SIGKILL)) + select { + case <-exited: + var exitErr *exec.ExitError + require.ErrorAs(t, waitErr, &exitErr) + assert.Equal(t, unix.SIGKILL, exitErr.Sys().(syscall.WaitStatus).Signal()) + case <-time.After(5 * time.Second): + t.Fatal("SIGKILL must end the init") + } +} + +// initCopy copies the test binary where the init user can run it: the +// directories go test builds it in are not traversable for other users. +func initCopy(t *testing.T) string { + self, err := os.Executable() + require.NoError(t, err) + dir := t.TempDir() + for d := dir; d != os.TempDir(); d = filepath.Dir(d) { + require.NoError(t, os.Chmod(d, 0o755)) + } + src, err := os.Open(self) + require.NoError(t, err) + defer src.Close() + path := filepath.Join(dir, "init") + dst, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0o755) + require.NoError(t, err) + _, err = io.Copy(dst, src) + require.NoError(t, errors.Join(err, dst.Close())) + return path +} + +// signalMasks reads the ignored and caught signal sets of pid from /proc; +// bit n-1 stands for signal n. +func signalMasks(t *testing.T, pid int) (ignored, caught uint64) { + data, err := os.ReadFile(fmt.Sprintf("/proc/%d/status", pid)) + require.NoError(t, err) + for _, line := range strings.Split(string(data), "\n") { + field, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + value = strings.TrimSpace(value) + switch field { + case "SigIgn": + ignored, err = strconv.ParseUint(value, 16, 64) + require.NoError(t, err) + case "SigCgt": + caught, err = strconv.ParseUint(value, 16, 64) + require.NoError(t, err) + } + } + return ignored, caught +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/main.go b/cmd/containerd-shim-sandboxed-runc-v2/main.go new file mode 100644 index 0000000000..00a963ea9a --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/main.go @@ -0,0 +1,36 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// containerd-shim-sandboxed-runc-v2 is an experimental runc shim that +// implements the Sandbox API itself, so that a pod needs no pause container: +// the shim holds the pod namespaces and owns the pod shared files, and runs +// the containers of the pod with the same runc task service as +// containerd-shim-runc-v2. It serves the io.containerd.runc.v2 runtime type; +// a runtime handler selects it with runtime_path and sandboxer = "shim". See +// the sandbox package for details on the design. +package main + +import ( + "context" + + "github.com/containerd/containerd/v2/pkg/shim" +) + +func main() { + shim.RunShim(context.Background(), newManager()) +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/manager.go b/cmd/containerd-shim-sandboxed-runc-v2/manager.go new file mode 100644 index 0000000000..cb9632de6d --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/manager.go @@ -0,0 +1,316 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + goruntime "runtime" + "slices" + "syscall" + + "github.com/containerd/cgroups/v3" + cgroupsv2 "github.com/containerd/cgroups/v3/cgroup2" + bootapi "github.com/containerd/containerd/api/runtime/bootstrap/v1" + "github.com/containerd/containerd/api/types" + "github.com/containerd/containerd/api/types/runc/options" + "github.com/containerd/errdefs" + "github.com/containerd/typeurl/v2" + "github.com/opencontainers/runtime-spec/specs-go/features" + + legacymanager "github.com/containerd/containerd/v2/cmd/containerd-shim-runc-v2/manager" + "github.com/containerd/containerd/v2/cmd/containerd-shim-runc-v2/runc" + "github.com/containerd/containerd/v2/cmd/containerd-shim-sandboxed-runc-v2/sandbox" + "github.com/containerd/containerd/v2/defaults" + "github.com/containerd/containerd/v2/pkg/namespaces" + "github.com/containerd/containerd/v2/pkg/schedcore" + "github.com/containerd/containerd/v2/pkg/shim" + "github.com/containerd/containerd/v2/plugins" +) + +// manager is a fork of the containerd-shim-runc-v2 manager for the sandbox +// flow: "start" runs once per pod, for the sandbox, keyed by the sandbox id +// (containers join the shim through the sandbox endpoint, so there is no +// grouping by OCI annotations); "delete" also cleans up a sandbox bundle; and +// "-info" hides the user namespace feature. +// +// TODO(mxpv): consolidate with the containerd-shim-runc-v2 manager. +type manager struct { + // legacy is the containerd-shim-runc-v2 manager. It serves the runtime + // info (the same runtime type, the same runc, the same features) and the + // cleanup of the task bundles joined to this shim. + legacy shim.Shim +} + +func newManager() shim.Shim { + return &manager{legacy: legacymanager.NewShimManager(plugins.RuntimeRuncV2)} +} + +// Name is the runtime type served: containerd reaches this binary through +// runtime_path, not through a runtime type of its own. +func (m *manager) Name() string { + return m.legacy.Name() +} + +func newCommand(ctx context.Context, id, containerdAddress string, debug bool) (*exec.Cmd, error) { + ns, err := namespaces.NamespaceRequired(ctx) + if err != nil { + return nil, err + } + self, err := os.Executable() + if err != nil { + return nil, err + } + cwd, err := os.Getwd() + if err != nil { + return nil, err + } + args := []string{ + "-namespace", ns, + "-id", id, + "-address", containerdAddress, + } + if debug { + args = append(args, "-debug") + } + cmd := exec.Command(self, args...) + cmd.Dir = cwd + cmd.Env = append(os.Environ(), "GOMAXPROCS=4") + cmd.Env = append(cmd.Env, "OTEL_SERVICE_NAME=containerd-shim-"+id) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, + } + return cmd, nil +} + +type shimSocket struct { + addr string + s *net.UnixListener + f *os.File +} + +func (s *shimSocket) Close() { + if s.s != nil { + s.s.Close() + } + if s.f != nil { + s.f.Close() + } + _ = shim.RemoveSocket(s.addr) +} + +func newShimSocket(ctx context.Context, root, path, id string, debug bool) (*shimSocket, error) { + address, err := shim.CreateSocketAddress(ctx, root, path, id, debug) + if err != nil { + return nil, err + } + socket, err := shim.NewSocket(address) + if err != nil { + // A socket in use that accepts connections belongs to a shim that is + // already running for this sandbox; a stale one is replaced. + if !shim.SocketEaddrinuse(err) { + return nil, fmt.Errorf("create new shim socket: %w", err) + } + if !debug && shim.CanConnect(address) { + return &shimSocket{addr: address}, errdefs.ErrAlreadyExists + } + if err := shim.RemoveSocket(address); err != nil { + return nil, fmt.Errorf("remove pre-existing socket: %w", err) + } + if socket, err = shim.NewSocket(address); err != nil { + return nil, fmt.Errorf("try create new shim socket 2x: %w", err) + } + } + s := &shimSocket{ + addr: address, + s: socket, + } + f, err := socket.File() + if err != nil { + s.Close() + return nil, err + } + s.f = f + return s, nil +} + +// Start starts the shim process for a sandbox bundle. containerd calls it once +// per sandbox; the containers of the pod join the running shim through the +// bootstrap protocol without invoking "start" again. +func (m *manager) Start(ctx context.Context, opts *bootapi.BootstrapParams) (_ *bootapi.BootstrapResult, retErr error) { + if cgroups.Mode() != cgroups.Unified { + return nil, fmt.Errorf("%s requires cgroup v2 (the unified hierarchy) and this host runs cgroup v1: %w", sandbox.BinaryName, errdefs.ErrNotImplemented) + } + + // The working directory is the sandbox bundle. + cwd, err := os.Getwd() + if err != nil { + return nil, err + } + + var params bootapi.BootstrapResult + params.Version = 3 + params.Protocol = "ttrpc" + + id := opts.GetInstanceID() + debugLog := opts.GetLogLevel() <= bootapi.LogLevel_LOG_LEVEL_DEBUG + + cmd, err := newCommand(ctx, id, opts.GetContainerdGrpcAddress(), debugLog) + if err != nil { + return nil, err + } + + var sockets []*shimSocket + defer func() { + if retErr != nil { + for _, s := range sockets { + s.Close() + } + } + }() + + socketDir := opts.GetSocketDir() + if socketDir == "" { + socketDir = filepath.Join(defaults.DefaultStateDir, "s") + } + // The socket is keyed by the sandbox id alone. + s, err := newShimSocket(ctx, socketDir, opts.GetContainerdGrpcAddress(), id, false) + if err != nil { + if errdefs.IsAlreadyExists(err) { + params.Address = s.addr + return ¶ms, nil + } + return nil, err + } + sockets = append(sockets, s) + cmd.ExtraFiles = append(cmd.ExtraFiles, s.f) + + if debugLog { + s, err = newShimSocket(ctx, socketDir, opts.GetContainerdGrpcAddress(), id, true) + if err != nil { + return nil, err + } + sockets = append(sockets, s) + cmd.ExtraFiles = append(cmd.ExtraFiles, s.f) + } + + // The shim is spawned from a locked thread so that a core scheduling + // cookie created here is inherited by it; the thread is released on + // every path. + goruntime.LockOSThread() + err = func() error { + if os.Getenv("SCHED_CORE") != "" { + if err := schedcore.Create(schedcore.ProcessGroup); err != nil { + return fmt.Errorf("enable sched core support: %w", err) + } + } + return cmd.Start() + }() + goruntime.UnlockOSThread() + if err != nil { + return nil, err + } + + defer func() { + if retErr != nil { + cmd.Process.Kill() + } + }() + // make sure to wait after start + go cmd.Wait() + + var runcOpts options.Options + if found, err := opts.FindExtension(&runcOpts); err != nil { + return nil, fmt.Errorf("failed to fetch runc options: %w", err) + } else if found { + // The runc options are persisted in the bundle. The sandbox service + // reads them to run containerd-sandbox-init through the same runc, + // root and cgroup driver as the containers of the pod, and Stop + // reads them to delete that container once the shim is gone. + if err := runc.WriteOptions(cwd, &runcOpts); err != nil { + return nil, fmt.Errorf("failed to persist the runc options: %w", err) + } + if err := runc.WriteRuntime(cwd, runcOpts.GetBinaryName()); err != nil { + return nil, fmt.Errorf("failed to persist the runtime name: %w", err) + } + if shimCgroup := runcOpts.GetShimCgroup(); shimCgroup != "" { + cg, err := cgroupsv2.Load(shimCgroup) + if err != nil { + return nil, fmt.Errorf("failed to load cgroup %s: %w", shimCgroup, err) + } + if err := cg.AddProc(uint64(cmd.Process.Pid)); err != nil { + return nil, fmt.Errorf("failed to join cgroup %s: %w", shimCgroup, err) + } + } + } + + if err := shim.AdjustOOMScore(cmd.Process.Pid); err != nil { + return nil, fmt.Errorf("failed to adjust OOM score for shim: %w", err) + } + + params.Address = sockets[0].addr + return ¶ms, nil +} + +// Stop is "shim delete". containerd runs it in a bundle once the shim that +// served the bundle is gone: in the sandbox bundle, and in the bundle of every +// container task that joined the shim. Stop does not tell the two apart and +// runs both cleanups every time: +// - The runc task cleanup of containerd-shim-runc-v2 force-deletes the runc +// container of that id and detaches its rootfs. In the sandbox bundle of a +// shared-PID pod that deletes containerd-sandbox-init; in any other +// sandbox bundle it does nothing. +// - The sandbox cleanup detaches every mount under the bundle and removes +// the pod files. In a task bundle it does nothing. +func (m *manager) Stop(ctx context.Context, id string) (shim.StopStatus, error) { + status, err := m.legacy.Stop(ctx, id) + cwd, cwdErr := os.Getwd() + if cwdErr != nil { + return status, errors.Join(err, cwdErr) + } + return status, errors.Join(err, sandbox.Cleanup(ctx, filepath.Join(filepath.Dir(cwd), id))) +} + +// Info returns the runtime info of containerd-shim-runc-v2: the same runtime +// type, and the features of the same runc minus the user namespace. CRI +// advertises user namespace support for a handler from that feature, and this +// shim rejects user namespaced pods for now. +func (m *manager) Info(ctx context.Context, optionsR io.Reader) (*types.RuntimeInfo, error) { + info, err := m.legacy.Info(ctx, optionsR) + if err != nil || info.GetFeatures() == nil { + return info, err + } + var feat features.Features + if err := typeurl.UnmarshalTo(info.Features, &feat); err != nil { + return nil, fmt.Errorf("failed to decode the runc features: %w", err) + } + if feat.Linux != nil { + feat.Linux.Namespaces = slices.DeleteFunc(feat.Linux.Namespaces, func(ns string) bool { return ns == "user" }) + } + if info.Features, err = typeurl.MarshalAnyToProto(&feat); err != nil { + return nil, fmt.Errorf("failed to encode the runc features: %w", err) + } + return info, nil +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/manager_test.go b/cmd/containerd-shim-sandboxed-runc-v2/manager_test.go new file mode 100644 index 0000000000..db3b171c8e --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/manager_test.go @@ -0,0 +1,61 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/containerd/containerd/v2/pkg/namespaces" + "github.com/containerd/containerd/v2/plugins" +) + +func TestManagerName(t *testing.T) { + assert.Equal(t, plugins.RuntimeRuncV2, newManager().Name()) +} + +func TestStopCleansUpTheBundle(t *testing.T) { + // "shim delete" in a sandbox bundle removes what the sandbox created and + // leaves what containerd created. The runc task cleanup that runs first + // finds no container there and only logs about it. + root := t.TempDir() + const id = "sandbox-id" + bundle := filepath.Join(root, id) + require.NoError(t, os.MkdirAll(filepath.Join(bundle, "ns"), 0o700)) + require.NoError(t, os.Mkdir(filepath.Join(bundle, "rootfs"), 0o700)) + for _, f := range []string{"hostname", "hosts", "resolv.conf", filepath.Join("ns", "ipc"), filepath.Join("ns", "uts")} { + require.NoError(t, os.WriteFile(filepath.Join(bundle, f), nil, 0o600)) + } + t.Chdir(bundle) + + _, err := newManager().Stop(namespaces.WithNamespace(context.Background(), "test"), id) + require.NoError(t, err) + + for _, f := range []string{"hostname", "hosts", "resolv.conf", "ns"} { + _, err := os.Stat(filepath.Join(bundle, f)) + assert.True(t, os.IsNotExist(err), "%s should be gone", f) + } + _, err = os.Stat(filepath.Join(bundle, "rootfs")) + assert.NoError(t, err, "what containerd created is left alone") +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle.go new file mode 100644 index 0000000000..206c312b6c --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle.go @@ -0,0 +1,151 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/moby/sys/mountinfo" + specs "github.com/opencontainers/runtime-spec/specs-go" + "golang.org/x/sys/unix" + + "github.com/containerd/containerd/v2/core/mount" + "github.com/containerd/containerd/v2/internal/cri/sandboxfiles" +) + +// Everything the sandbox creates lives under the shim bundle at fixed paths +// (see the package doc): +// +// /ns/{ipc,uts} namespace pins, and ns/pid for a shared-PID pod +// /hostname the pod hostname +// /hosts a copy of the host /etc/hosts +// /resolv.conf the pod DNS configuration +// /shm the pod /dev/shm tmpfs +// /rootfs the tmpfs rootfs of containerd-sandbox-init, with +// its config.json and init.pid beside (shared-PID pods) + +// setupFiles creates the pod shared files and returns the mounts that expose +// them to the containers of the pod; CRI picks the sources up from the +// sandbox spec. +func setupFiles(bundle string, cfg *Config) ([]specs.Mount, error) { + var mounts []specs.Mount + bind := func(destination, source string) { + mounts = append(mounts, specs.Mount{ + Destination: destination, + Type: "bind", + Source: source, + Options: []string{"rbind"}, + }) + } + + hostname := filepath.Join(bundle, sandboxfiles.HostnameFile) + if err := os.WriteFile(hostname, sandboxfiles.HostnameContent(cfg.Hostname), 0o644); err != nil { + return nil, fmt.Errorf("failed to write %s: %w", hostname, err) + } + bind(sandboxfiles.EtcHostname, hostname) + + hosts := filepath.Join(bundle, sandboxfiles.HostsFile) + if err := copyFile(sandboxfiles.EtcHosts, hosts); err != nil { + return nil, fmt.Errorf("failed to create %s: %w", hosts, err) + } + bind(sandboxfiles.EtcHosts, hosts) + + resolvConf := filepath.Join(bundle, sandboxfiles.ResolvConfFile) + if cfg.DNS != nil { + content := sandboxfiles.ResolvConfContent(cfg.DNS.GetServers(), cfg.DNS.GetSearches(), cfg.DNS.GetOptions()) + if err := os.WriteFile(resolvConf, content, 0o644); err != nil { + return nil, fmt.Errorf("failed to write %s: %w", resolvConf, err) + } + } else { + // No DNS config means the host resolver configuration, as for pause. + if err := copyFile(sandboxfiles.ResolvConfPath, resolvConf); err != nil { + return nil, fmt.Errorf("failed to create %s: %w", resolvConf, err) + } + } + bind(sandboxfiles.ResolvConfPath, resolvConf) + + if !cfg.HostIPC { + shm := filepath.Join(bundle, sandboxfiles.ShmDir) + if err := os.Mkdir(shm, 0o700); err != nil && !errors.Is(err, fs.ErrExist) { + return nil, fmt.Errorf("failed to create %s: %w", shm, err) + } + if err := unix.Mount("shm", shm, "tmpfs", unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV, sandboxfiles.ShmMountData(cfg.ShmSize)); err != nil { + return nil, fmt.Errorf("failed to mount the pod shm at %s: %w", shm, err) + } + bind(sandboxfiles.DevShm, shm) + } + return mounts, nil +} + +func copyFile(src, dst string) error { + b, err := os.ReadFile(src) + if err != nil { + return err + } + return os.WriteFile(dst, b, 0o644) +} + +// Cleanup removes everything the sandbox shim created under bundle: it +// detaches the mounts at their fixed paths (the namespace pins, the shm tmpfs +// and the init rootfs) and removes the pin and shared files. It is idempotent +// and relies on no state, so it serves create failure, StopSandbox and "shim +// delete" alike; a mount left behind would keep containerd from deleting the +// bundle. The init container itself is deleted through runc by its owner +// (initProcess.stop, or the runc task cleanup of "shim delete"). +func Cleanup(ctx context.Context, bundle string) error { + var errs []error + for _, p := range []string{ + filepath.Join(bundle, pinDir, ipcPin), + filepath.Join(bundle, pinDir, utsPin), + filepath.Join(bundle, pinDir, pidPin), + filepath.Join(bundle, sandboxfiles.ShmDir), + filepath.Join(bundle, initRootfs), + } { + // Only a mount point is unmounted: umount(2) on a plain file needs + // the same privilege as on a mount, and answers EPERM before EINVAL. + mounted, err := mountinfo.Mounted(p) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + errs = append(errs, fmt.Errorf("failed to check %s: %w", p, err)) + } + continue + } + if !mounted { + continue + } + if err := mount.UnmountAll(p, unix.MNT_DETACH); err != nil { + errs = append(errs, fmt.Errorf("failed to detach %s: %w", p, err)) + } + } + if len(errs) > 0 { + // Nothing is removed while one of the mounts is still in place. + return errors.Join(errs...) + } + for _, name := range []string{pinDir, sandboxfiles.ShmDir, sandboxfiles.HostnameFile, sandboxfiles.HostsFile, sandboxfiles.ResolvConfFile} { + if err := os.RemoveAll(filepath.Join(bundle, name)); err != nil { + errs = append(errs, fmt.Errorf("failed to remove %s: %w", name, err)) + } + } + return errors.Join(errs...) +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle_test.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle_test.go new file mode 100644 index 0000000000..bb5dc2e06c --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/bundle_test.go @@ -0,0 +1,128 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "os" + "path/filepath" + "testing" + + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" + criruntime "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/containerd/containerd/v2/pkg/testutil" +) + +func TestSetupFiles(t *testing.T) { + testutil.RequiresRoot(t) + ctx := context.Background() + bundle := t.TempDir() + t.Cleanup(func() { _ = Cleanup(ctx, bundle) }) + + cfg := &Config{ + ID: testID, + Hostname: "pod-hostname", + DNS: &criruntime.DNSConfig{ + Servers: []string{"10.0.0.10"}, + Searches: []string{"svc.cluster.local"}, + Options: []string{"ndots:5"}, + }, + ShmSize: 1 << 20, + } + mounts, err := setupFiles(bundle, cfg) + require.NoError(t, err) + + byDestination := make(map[string]specs.Mount) + for _, m := range mounts { + byDestination[m.Destination] = m + } + require.Len(t, byDestination, 4) + + content, err := os.ReadFile(byDestination["/etc/hostname"].Source) + require.NoError(t, err) + assert.Equal(t, "pod-hostname\n", string(content)) + + content, err = os.ReadFile(byDestination["/etc/resolv.conf"].Source) + require.NoError(t, err) + assert.Equal(t, "search svc.cluster.local\nnameserver 10.0.0.10\noptions ndots:5\n", string(content)) + + hostHosts, err := os.ReadFile("/etc/hosts") + require.NoError(t, err) + content, err = os.ReadFile(byDestination["/etc/hosts"].Source) + require.NoError(t, err) + assert.Equal(t, string(hostHosts), string(content), "the pod hosts file starts as a copy of the host one") + + var st unix.Statfs_t + require.NoError(t, unix.Statfs(byDestination["/dev/shm"].Source, &st)) + assert.EqualValues(t, unix.TMPFS_MAGIC, st.Type, "the pod shm is a tmpfs") + assert.EqualValues(t, 1<<20, st.Blocks*uint64(st.Bsize), "sized as requested") + + require.NoError(t, Cleanup(ctx, bundle)) + assertNoMountsUnder(t, bundle) + entries, err := os.ReadDir(bundle) + require.NoError(t, err) + assert.Empty(t, entries) + + // With the host IPC namespace the pod uses the host /dev/shm: no tmpfs. + cfg.HostIPC = true + cfg.DNS = nil + mounts, err = setupFiles(bundle, cfg) + require.NoError(t, err) + require.Len(t, mounts, 3) + for _, m := range mounts { + assert.NotEqual(t, "/dev/shm", m.Destination) + } + hostResolv, err := os.ReadFile("/etc/resolv.conf") + require.NoError(t, err) + content, err = os.ReadFile(filepath.Join(bundle, "resolv.conf")) + require.NoError(t, err) + assert.Equal(t, string(hostResolv), string(content), "no DNS config means the host resolver") +} + +func TestCleanupWithoutMounts(t *testing.T) { + ctx := context.Background() + bundle := t.TempDir() + + // Leftovers of a sandbox whose mounts are already gone (or that crashed + // before mounting anything) are removed, and nothing else is touched. + require.NoError(t, os.Mkdir(filepath.Join(bundle, pinDir), 0o700)) + require.NoError(t, os.Mkdir(filepath.Join(bundle, "shm"), 0o700)) + require.NoError(t, os.Mkdir(filepath.Join(bundle, "rootfs"), 0o700)) + for _, f := range []string{"hostname", "hosts", "resolv.conf", "ns/ipc", "ns/uts", "address"} { + require.NoError(t, os.WriteFile(filepath.Join(bundle, f), []byte("x"), 0o600)) + } + + require.NoError(t, Cleanup(ctx, bundle)) + + entries, err := os.ReadDir(bundle) + require.NoError(t, err) + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + assert.ElementsMatch(t, []string{"address", "rootfs"}, names, "only what containerd owns is left") + + // Idempotent, including on a bundle that no longer exists. + require.NoError(t, Cleanup(ctx, bundle)) + require.NoError(t, Cleanup(ctx, filepath.Join(bundle, "does-not-exist"))) +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config.go new file mode 100644 index 0000000000..bba2c7637e --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config.go @@ -0,0 +1,288 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "fmt" + "os" + "strings" + + api "github.com/containerd/containerd/api/runtime/sandbox/v1" + "github.com/containerd/errdefs" + "github.com/containerd/typeurl/v2" + specs "github.com/opencontainers/runtime-spec/specs-go" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/containerd/containerd/v2/internal/cri/sandboxfiles" +) + +// Config is what the sandbox shim needs to know about a pod. It is derived +// from the CRI PodSandboxConfig that CRI passes as the sandbox options. +type Config struct { + // ID is the sandbox id. + ID string + // Hostname is the pod hostname, the node hostname when the pod sets none. + Hostname string + // HostNetwork means the pod uses the host network and UTS namespaces; the + // sandbox then holds neither and NetNSPath is empty. + HostNetwork bool + // HostIPC means the pod uses the host IPC namespace and the host /dev/shm. + HostIPC bool + // HostPID means the containers of the pod use the host PID namespace. + HostPID bool + // SharedPID means the pod asked for one PID namespace shared by its + // containers (shareProcessNamespace); containerd-sandbox-init is then + // run as its PID 1 (see initProcess). + SharedPID bool + // NetNSPath is the network namespace CRI created for the pod. + NetNSPath string + // Sysctls is the effective set of namespaced sysctls to apply. + Sysctls map[string]string + // CgroupParent is the pod cgroup as a cgroup v2 group path: the sandbox + // reports its metrics, and places the init of a shared-PID pod under it. + // kubelet spells it that way with both cgroup drivers: the systemd + // driver hands over the expanded slice path, which CRI shortens to its + // base for runc. + CgroupParent string + // DNS is the pod DNS configuration; nil means the host resolv.conf. + DNS *runtime.DNSConfig + // ShmSize is the size of the pod /dev/shm tmpfs. + ShmSize int64 + // Annotations are the annotations of the synthesized sandbox spec. + Annotations map[string]string +} + +// ConfigFromRequest derives the sandbox configuration from a create request. +// It rejects what this shim does not support yet with ErrNotImplemented so the +// error is explicit rather than a pod that silently lacks isolation. +func ConfigFromRequest(r *api.CreateSandboxRequest) (*Config, error) { + id := r.GetSandboxID() + if r.GetOptions() == nil { + return nil, fmt.Errorf("sandbox %q: no options; %s expects the CRI PodSandboxConfig: %w", id, BinaryName, errdefs.ErrInvalidArgument) + } + // TODO: define a containerd-owned sandbox configuration message for the + // handoff (hostname, namespace modes, the effective sysctls including the + // enable_unprivileged_ports/enable_unprivileged_icmp defaults CRI applies + // to pause, DNS, cgroup parent, labels) and have CRI send it as an + // extension of the create request; a sandboxed shim should not depend on + // the CRI layer and its PodSandboxConfig, nor re-derive CRI policy. + var pc runtime.PodSandboxConfig + if err := typeurl.UnmarshalTo(r.GetOptions(), &pc); err != nil { + return nil, fmt.Errorf("sandbox %q: options are not a CRI PodSandboxConfig: %w", id, err) + } + return configFromPodSandboxConfig(id, r.GetNetnsPath(), r.GetAnnotations(), &pc) +} + +func configFromPodSandboxConfig(id, netns string, reqAnnotations map[string]string, pc *runtime.PodSandboxConfig) (*Config, error) { + nsOpts := pc.GetLinux().GetSecurityContext().GetNamespaceOptions() + cfg := &Config{ + ID: id, + Hostname: pc.GetHostname(), + HostNetwork: nsOpts.GetNetwork() == runtime.NamespaceMode_NODE, + HostIPC: nsOpts.GetIpc() == runtime.NamespaceMode_NODE, + HostPID: nsOpts.GetPid() == runtime.NamespaceMode_NODE, + SharedPID: nsOpts.GetPid() == runtime.NamespaceMode_POD, + NetNSPath: netns, + CgroupParent: pc.GetLinux().GetCgroupParent(), + DNS: pc.GetDnsConfig(), + ShmSize: sandboxfiles.DefaultShmSize, + } + + if nsOpts.GetPid() == runtime.NamespaceMode_TARGET { + return nil, fmt.Errorf("sandbox %q: PID namespace mode TARGET is not valid for a pod sandbox: %w", id, errdefs.ErrInvalidArgument) + } + if u := nsOpts.GetUsernsOptions(); u != nil && u.GetMode() != runtime.NamespaceMode_NODE { + // The namespaces of a user namespaced pod must be owned by the pod + // user namespace, which a multithreaded shim cannot enter; CRI has + // to create them. That is a follow-up. + return nil, fmt.Errorf("sandbox %q: user namespaces are not supported yet by %s: %w", id, BinaryName, errdefs.ErrNotImplemented) + } + + if cfg.HostNetwork { + cfg.NetNSPath = "" + } else if cfg.NetNSPath == "" { + return nil, fmt.Errorf("sandbox %q: no network namespace path for a pod with its own network namespace: %w", id, errdefs.ErrInvalidArgument) + } + + if cfg.Hostname == "" { + hostname, err := os.Hostname() + if err != nil { + return nil, fmt.Errorf("sandbox %q: failed to get the node hostname: %w", id, err) + } + cfg.Hostname = hostname + } + + sysctls, err := effectiveSysctls(pc.GetLinux().GetSysctls(), cfg.HostNetwork, cfg.HostIPC) + if err != nil { + return nil, fmt.Errorf("sandbox %q: %w", id, err) + } + cfg.Sysctls = sysctls + cfg.Annotations = reqAnnotations + return cfg, nil +} + +// Sysctls are applied by the shim thread that created the pod namespaces (for +// pause, runc applied them inside the pause container), so only namespaced +// sysctls of namespaces the pod owns can be honored. + +const sysctlRoot = "/proc/sys" + +// defaultSysctls are applied to pods with their own network namespace unless +// the pod sets them, matching what containerd applies to pause sandboxes with +// its default CRI configuration (enable_unprivileged_ports and +// enable_unprivileged_icmp both on). +var defaultSysctls = map[string]string{ + "net.ipv4.ip_unprivileged_port_start": "0", + "net.ipv4.ping_group_range": "0 2147483647", +} + +// validateSysctl accepts the namespaced sysctls the shim can apply from a host +// thread that entered the pod namespaces, and rejects the rest: user.* is +// scoped to a user namespace the shim never enters, a sysctl of a namespace +// the pod shares with the host would change the host, and anything else is +// not namespaced at all. It returns the key in dotted form, the form it +// classified and the one that names the file (see sysctlPath). +func validateSysctl(key string, hostNetwork, hostIPC bool) (string, error) { + _, dotted, err := sysctlPath(key) + if err != nil { + return "", err + } + switch { + case strings.HasPrefix(dotted, "net."): + if hostNetwork { + return "", fmt.Errorf("sysctl %q is not allowed for a pod using the host network: %w", key, errdefs.ErrInvalidArgument) + } + case strings.HasPrefix(dotted, "kernel.shm"), strings.HasPrefix(dotted, "kernel.msg"), dotted == "kernel.sem", strings.HasPrefix(dotted, "fs.mqueue."): + if hostIPC { + return "", fmt.Errorf("sysctl %q is not allowed for a pod using the host IPC namespace: %w", key, errdefs.ErrInvalidArgument) + } + case dotted == "kernel.hostname", dotted == "kernel.domainname": + if hostNetwork { + return "", fmt.Errorf("sysctl %q is not allowed for a pod using the host network (and UTS) namespace: %w", key, errdefs.ErrInvalidArgument) + } + case strings.HasPrefix(dotted, "user."): + return "", fmt.Errorf("sysctl %q: user.* sysctls are not supported by %s: %w", key, BinaryName, errdefs.ErrNotImplemented) + default: + return "", fmt.Errorf("sysctl %q is not namespaced and cannot be set for a pod: %w", key, errdefs.ErrInvalidArgument) + } + return dotted, nil +} + +// effectiveSysctls validates the pod sysctls and adds the defaults. The keys +// of the result are in dotted form. +func effectiveSysctls(pod map[string]string, hostNetwork, hostIPC bool) (map[string]string, error) { + out := make(map[string]string, len(pod)+len(defaultSysctls)) + for k, v := range pod { + dotted, err := validateSysctl(k, hostNetwork, hostIPC) + if err != nil { + return nil, err + } + out[dotted] = v + } + if !hostNetwork { + for k, v := range defaultSysctls { + if _, ok := out[k]; !ok { + out[k] = v + } + } + } + return out, nil +} + +// sysctlPath maps a sysctl key to its file under /proc/sys and returns the key +// in dotted form, the form validateSysctl classifies. Keys are spelled as in +// sysctl.d(5), which is what kubelet passes through and runc accepts: the +// first separator, "." or "/", separates the components and the other +// character is literal. The rp_filter of the interface eth0.100 is therefore +// "net.ipv4.conf.eth0/100.rp_filter" or "net/ipv4/conf/eth0.100/rp_filter", +// the file /proc/sys/net/ipv4/conf/eth0.100/rp_filter. Every component must +// be a name: with "." or ".." the file would be of another class than the +// dotted form says. +func sysctlPath(key string) (path, dotted string, err error) { + swap := func(r rune) rune { + switch r { + case '.': + return '/' + case '/': + return '.' + } + return r + } + dotted = key + if i := strings.IndexAny(key, "./"); i >= 0 && key[i] == '/' { + dotted = strings.Map(swap, key) + } + rel := strings.Map(swap, dotted) + for c := range strings.SplitSeq(rel, "/") { + if c == "" || c == "." || c == ".." || strings.ContainsRune(c, 0) { + return "", "", fmt.Errorf("invalid sysctl key %q: %w", key, errdefs.ErrInvalidArgument) + } + } + return sysctlRoot + "/" + rel, dotted, nil +} + +// buildSpec synthesizes the sandbox spec CRI and NRI consume. It is not the +// spec of a process: it describes the namespaces the sandbox holds, from what +// was actually pinned, the sysctls applied to them, and the pod shared files +// as mounts. A namespace the pod shares with the host has no entry. Linux is +// always set, which is how CRI tells a sandbox without a process from an +// older Sandbox API shim that returns no spec. +// +// The PID namespace entry has a path only for a pod that shares its PID +// namespace (NamespaceMode_POD, the CRI default and kubelet's +// shareProcessNamespace): pidNS, the pinned namespace containerd-sandbox-init +// is PID 1 of. Otherwise the sandbox holds no PID namespace, and the entry, without a +// path, declares that each container gets a new one of its own; CRI honors +// the declaration. A host PID pod has no entry. +func buildSpec(cfg *Config, pins Pins, mounts []specs.Mount, pidNS string) *specs.Spec { + s := &specs.Spec{ + Version: specs.Version, + Hostname: cfg.Hostname, + Annotations: cfg.Annotations, + Mounts: mounts, + Linux: &specs.Linux{ + Sysctl: cfg.Sysctls, + Namespaces: podNamespaces(cfg, pins), + }, + } + switch { + case cfg.HostPID: + case pidNS != "": + s.Linux.Namespaces = append(s.Linux.Namespaces, specs.LinuxNamespace{Type: specs.PIDNamespace, Path: pidNS}) + default: + s.Linux.Namespaces = append(s.Linux.Namespaces, specs.LinuxNamespace{Type: specs.PIDNamespace}) + } + return s +} + +// podNamespaces are the network, IPC and UTS namespaces the sandbox holds, +// which the sandbox spec lists and containerd-sandbox-init joins. +func podNamespaces(cfg *Config, pins Pins) []specs.LinuxNamespace { + var ns []specs.LinuxNamespace + if cfg.NetNSPath != "" { + ns = append(ns, specs.LinuxNamespace{Type: specs.NetworkNamespace, Path: cfg.NetNSPath}) + } + if pins.IPC != "" { + ns = append(ns, specs.LinuxNamespace{Type: specs.IPCNamespace, Path: pins.IPC}) + } + if pins.UTS != "" { + ns = append(ns, specs.LinuxNamespace{Type: specs.UTSNamespace, Path: pins.UTS}) + } + return ns +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config_test.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config_test.go new file mode 100644 index 0000000000..53c25bddc7 --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/config_test.go @@ -0,0 +1,371 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "os" + "testing" + + api "github.com/containerd/containerd/api/runtime/sandbox/v1" + "github.com/containerd/errdefs" + "github.com/containerd/typeurl/v2" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" +) + +const ( + testID = "sandbox-id" + testNetNS = "/var/run/netns/cni-test" +) + +func testPodConfig() *runtime.PodSandboxConfig { + return &runtime.PodSandboxConfig{ + Metadata: &runtime.PodSandboxMetadata{ + Name: "pod", + Uid: "pod-uid", + Namespace: "pod-ns", + }, + Hostname: "pod-hostname", + LogDirectory: "/var/log/pods/pod", + DnsConfig: &runtime.DNSConfig{ + Servers: []string{"10.0.0.10"}, + }, + Linux: &runtime.LinuxPodSandboxConfig{ + CgroupParent: "/kubepods/pod", + SecurityContext: &runtime.LinuxSandboxSecurityContext{ + NamespaceOptions: &runtime.NamespaceOption{}, + }, + }, + } +} + +func createRequest(t *testing.T, pc *runtime.PodSandboxConfig, netns string) *api.CreateSandboxRequest { + t.Helper() + req := &api.CreateSandboxRequest{ + SandboxID: testID, + BundlePath: "/bundle", + NetnsPath: netns, + Annotations: map[string]string{"extra": "annotation"}, + } + if pc != nil { + any, err := typeurl.MarshalAnyToProto(pc) + require.NoError(t, err) + req.Options = any + } + return req +} + +func TestConfigFromRequest(t *testing.T) { + t.Run("options are required", func(t *testing.T) { + _, err := ConfigFromRequest(createRequest(t, nil, testNetNS)) + require.Error(t, err) + assert.True(t, errdefs.IsInvalidArgument(err)) + }) + + t.Run("pod namespaces", func(t *testing.T) { + cfg, err := ConfigFromRequest(createRequest(t, testPodConfig(), testNetNS)) + require.NoError(t, err) + assert.Equal(t, testID, cfg.ID) + assert.Equal(t, "pod-hostname", cfg.Hostname) + assert.False(t, cfg.HostNetwork) + assert.False(t, cfg.HostIPC) + assert.False(t, cfg.HostPID) + assert.Equal(t, testNetNS, cfg.NetNSPath) + assert.Equal(t, "/kubepods/pod", cfg.CgroupParent) + assert.Equal(t, []string{"10.0.0.10"}, cfg.DNS.GetServers()) + assert.Equal(t, defaultSysctls, cfg.Sysctls, "the pause defaults apply to a pod with its own network namespace") + assert.Equal(t, map[string]string{"extra": "annotation"}, cfg.Annotations, "the annotations of the request") + }) + + t.Run("pod network requires a network namespace path", func(t *testing.T) { + _, err := ConfigFromRequest(createRequest(t, testPodConfig(), "")) + require.Error(t, err) + assert.True(t, errdefs.IsInvalidArgument(err)) + }) + + t.Run("host network", func(t *testing.T) { + pc := testPodConfig() + pc.Linux.SecurityContext.NamespaceOptions.Network = runtime.NamespaceMode_NODE + cfg, err := ConfigFromRequest(createRequest(t, pc, "")) + require.NoError(t, err) + assert.True(t, cfg.HostNetwork) + assert.Empty(t, cfg.NetNSPath) + assert.Empty(t, cfg.Sysctls, "no network defaults for a pod in the host network namespace") + + pc.Linux.Sysctls = map[string]string{"net.ipv4.ip_forward": "1"} + _, err = ConfigFromRequest(createRequest(t, pc, "")) + require.Error(t, err, "a net sysctl would change the host") + assert.True(t, errdefs.IsInvalidArgument(err)) + }) + + t.Run("host ipc", func(t *testing.T) { + pc := testPodConfig() + pc.Linux.SecurityContext.NamespaceOptions.Ipc = runtime.NamespaceMode_NODE + cfg, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err) + assert.True(t, cfg.HostIPC) + + pc.Linux.Sysctls = map[string]string{"kernel.shm_rmid_forced": "1"} + _, err = ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.Error(t, err, "an IPC sysctl would change the host") + assert.True(t, errdefs.IsInvalidArgument(err)) + }) + + t.Run("host pid", func(t *testing.T) { + pc := testPodConfig() + pc.Linux.SecurityContext.NamespaceOptions.Pid = runtime.NamespaceMode_NODE + cfg, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err) + assert.True(t, cfg.HostPID) + }) + + t.Run("pod sysctls win over the defaults", func(t *testing.T) { + pc := testPodConfig() + pc.Linux.Sysctls = map[string]string{ + "net.ipv4.ip_unprivileged_port_start": "1024", + "kernel.shm_rmid_forced": "1", + "fs.mqueue.msg_max": "20", + } + cfg, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err) + assert.Equal(t, map[string]string{ + "net.ipv4.ip_unprivileged_port_start": "1024", + "net.ipv4.ping_group_range": "0 2147483647", + "kernel.shm_rmid_forced": "1", + "fs.mqueue.msg_max": "20", + }, cfg.Sysctls) + }) + + t.Run("slash separated sysctls", func(t *testing.T) { + // kubelet passes sysctl names with either separator through as + // written; they are classified and kept in dotted form. + pc := testPodConfig() + pc.Linux.Sysctls = map[string]string{"net/ipv4/ping_group_range": "1 1", "kernel/shm_rmid_forced": "1"} + cfg, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err) + assert.Equal(t, map[string]string{ + "net.ipv4.ip_unprivileged_port_start": "0", + "net.ipv4.ping_group_range": "1 1", + "kernel.shm_rmid_forced": "1", + }, cfg.Sysctls) + + pc.Linux.SecurityContext.NamespaceOptions.Ipc = runtime.NamespaceMode_NODE + _, err = ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.Error(t, err, "an IPC sysctl would change the host, however it is spelled") + assert.True(t, errdefs.IsInvalidArgument(err)) + }) + + t.Run("shared pid namespace is accepted but not shared", func(t *testing.T) { + // NamespaceMode_POD is the zero value, so it is what a pod config + // without explicit namespace options (crictl, critest) asks for. + pc := testPodConfig() + pc.Linux.SecurityContext.NamespaceOptions.Pid = runtime.NamespaceMode_POD + cfg, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err) + assert.True(t, cfg.SharedPID) + + pc.Linux.SecurityContext.NamespaceOptions.Pid = runtime.NamespaceMode_CONTAINER + cfg, err = ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err) + assert.False(t, cfg.SharedPID) + }) + + t.Run("target pid namespace is invalid for a sandbox", func(t *testing.T) { + pc := testPodConfig() + pc.Linux.SecurityContext.NamespaceOptions.Pid = runtime.NamespaceMode_TARGET + _, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.Error(t, err) + assert.True(t, errdefs.IsInvalidArgument(err)) + }) + + t.Run("user namespaces are not supported yet", func(t *testing.T) { + pc := testPodConfig() + pc.Linux.SecurityContext.NamespaceOptions.UsernsOptions = &runtime.UserNamespace{Mode: runtime.NamespaceMode_POD} + _, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.Error(t, err) + assert.True(t, errdefs.IsNotImplemented(err)) + + pc.Linux.SecurityContext.NamespaceOptions.UsernsOptions = &runtime.UserNamespace{Mode: runtime.NamespaceMode_NODE} + _, err = ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err, "the host user namespace is the default") + }) + + t.Run("unsupported sysctls", func(t *testing.T) { + pc := testPodConfig() + pc.Linux.Sysctls = map[string]string{"user.max_user_namespaces": "10"} + _, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.Error(t, err) + assert.True(t, errdefs.IsNotImplemented(err), "user.* needs the pod user namespace") + + pc.Linux.Sysctls = map[string]string{"vm.swappiness": "10"} + _, err = ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.Error(t, err) + assert.True(t, errdefs.IsInvalidArgument(err), "not namespaced") + + pc.Linux.Sysctls = map[string]string{".": "x"} + _, err = ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.Error(t, err) + assert.True(t, errdefs.IsInvalidArgument(err), "not a key") + }) + + t.Run("hostname defaults to the node hostname", func(t *testing.T) { + pc := testPodConfig() + pc.Hostname = "" + cfg, err := ConfigFromRequest(createRequest(t, pc, testNetNS)) + require.NoError(t, err) + hostname, err := os.Hostname() + require.NoError(t, err) + assert.Equal(t, hostname, cfg.Hostname) + }) +} + +func TestSysctlPath(t *testing.T) { + // Either separator, as in sysctl.d(5): the first one separates the + // components and the other character is literal, which is how a dotted + // interface name is spelled. The dotted form is what gets classified. + for key, want := range map[string][2]string{ + "net.ipv4.ip_unprivileged_port_start": {"/proc/sys/net/ipv4/ip_unprivileged_port_start", "net.ipv4.ip_unprivileged_port_start"}, + "net/ipv4/ip_unprivileged_port_start": {"/proc/sys/net/ipv4/ip_unprivileged_port_start", "net.ipv4.ip_unprivileged_port_start"}, + "net.ipv4.conf.eth0/100.rp_filter": {"/proc/sys/net/ipv4/conf/eth0.100/rp_filter", "net.ipv4.conf.eth0/100.rp_filter"}, + "net/ipv4/conf/eth0.100/rp_filter": {"/proc/sys/net/ipv4/conf/eth0.100/rp_filter", "net.ipv4.conf.eth0/100.rp_filter"}, + "kernel/shm_rmid_forced": {"/proc/sys/kernel/shm_rmid_forced", "kernel.shm_rmid_forced"}, + } { + p, dotted, err := sysctlPath(key) + require.NoError(t, err, "key %q", key) + assert.Equal(t, want[0], p, "key %q", key) + assert.Equal(t, want[1], dotted, "key %q", key) + } + + // No component may be empty, "." or "..": the file would not be under + // the directory the dotted form names. "net.//.kernel.shm_rmid_forced" + // (net/../kernel/shm_rmid_forced with slashes) is an IPC sysctl spelled + // as a net one. A NUL byte cannot be part of a path either. + for _, key := range []string{"", ".", "..", "/", "net.", ".net.ipv4.ip_forward", "net..ipv4.ip_forward", "net.//.kernel.shm_rmid_forced", "net/../kernel/shm_rmid_forced", "net./../kernel.shm_rmid_forced", "net.\x00"} { + _, _, err := sysctlPath(key) + assert.ErrorIs(t, err, errdefs.ErrInvalidArgument, "key %q", key) + } +} + +func TestValidateSysctl(t *testing.T) { + for _, tc := range []struct { + key string + hostNetwork, hostIPC bool + dotted string + }{ + {key: "net.ipv4.ip_forward", dotted: "net.ipv4.ip_forward"}, + {key: "kernel.shm_rmid_forced", dotted: "kernel.shm_rmid_forced"}, + {key: "kernel.sem", dotted: "kernel.sem"}, + {key: "fs.mqueue.msg_max", dotted: "fs.mqueue.msg_max"}, + {key: "kernel.hostname", dotted: "kernel.hostname"}, + {key: "kernel/shm_rmid_forced", dotted: "kernel.shm_rmid_forced"}, + {key: "net/ipv4/conf/eth0.100/rp_filter", dotted: "net.ipv4.conf.eth0/100.rp_filter"}, + // IPC sysctls are fine with the host network, net sysctls with the + // host IPC namespace. + {key: "kernel.shm_rmid_forced", hostNetwork: true, dotted: "kernel.shm_rmid_forced"}, + {key: "net.ipv4.ip_forward", hostIPC: true, dotted: "net.ipv4.ip_forward"}, + } { + dotted, err := validateSysctl(tc.key, tc.hostNetwork, tc.hostIPC) + require.NoError(t, err, "key %q", tc.key) + assert.Equal(t, tc.dotted, dotted, "key %q", tc.key) + } + + for _, tc := range []struct { + key string + hostNetwork, hostIPC bool + wantErr error + }{ + {key: "net.ipv4.ip_forward", hostNetwork: true, wantErr: errdefs.ErrInvalidArgument}, + {key: "net/ipv4/ip_forward", hostNetwork: true, wantErr: errdefs.ErrInvalidArgument}, + // The UTS namespace is shared with the host along with the network one. + {key: "kernel.hostname", hostNetwork: true, wantErr: errdefs.ErrInvalidArgument}, + {key: "kernel.domainname", hostNetwork: true, wantErr: errdefs.ErrInvalidArgument}, + {key: "kernel.shm_rmid_forced", hostIPC: true, wantErr: errdefs.ErrInvalidArgument}, + {key: "kernel/shm_rmid_forced", hostIPC: true, wantErr: errdefs.ErrInvalidArgument}, + {key: "kernel.msgmax", hostIPC: true, wantErr: errdefs.ErrInvalidArgument}, + {key: "fs.mqueue.msg_max", hostIPC: true, wantErr: errdefs.ErrInvalidArgument}, + // Like runc, only kernel.sem itself counts as an IPC sysctl. + {key: "kernel.sem_next_id", wantErr: errdefs.ErrInvalidArgument}, + {key: "user.max_user_namespaces", wantErr: errdefs.ErrNotImplemented}, + {key: "vm.swappiness", wantErr: errdefs.ErrInvalidArgument}, + {key: "net.\x00", wantErr: errdefs.ErrInvalidArgument}, + } { + _, err := validateSysctl(tc.key, tc.hostNetwork, tc.hostIPC) + assert.ErrorIs(t, err, tc.wantErr, "key %q, host network %v, host IPC %v", tc.key, tc.hostNetwork, tc.hostIPC) + } +} + +func TestBuildSpec(t *testing.T) { + pins := Pins{IPC: "/bundle/ns/ipc", UTS: "/bundle/ns/uts"} + mounts := []specs.Mount{{Destination: "/etc/hosts", Type: "bind", Source: "/bundle/hosts", Options: []string{"rbind"}}} + sysctls := map[string]string{"net.ipv4.ip_forward": "1"} + annots := map[string]string{"io.kubernetes.cri.sandbox-id": testID} + + t.Run("pod namespaces", func(t *testing.T) { + cfg := &Config{ID: testID, Hostname: "pod", NetNSPath: testNetNS, Sysctls: sysctls, Annotations: annots} + spec := buildSpec(cfg, pins, mounts, "") + assert.Equal(t, specs.Version, spec.Version) + assert.Equal(t, "pod", spec.Hostname) + assert.Equal(t, annots, spec.Annotations) + assert.Equal(t, mounts, spec.Mounts) + require.NotNil(t, spec.Linux) + assert.Equal(t, sysctls, spec.Linux.Sysctl) + assert.Equal(t, []specs.LinuxNamespace{ + {Type: specs.NetworkNamespace, Path: testNetNS}, + {Type: specs.IPCNamespace, Path: pins.IPC}, + {Type: specs.UTSNamespace, Path: pins.UTS}, + {Type: specs.PIDNamespace}, + }, spec.Linux.Namespaces) + }) + + t.Run("shared PID namespace", func(t *testing.T) { + cfg := &Config{ID: testID, Hostname: "pod", NetNSPath: testNetNS, SharedPID: true} + spec := buildSpec(cfg, pins, nil, "/proc/42/ns/pid") + assert.Equal(t, []specs.LinuxNamespace{ + {Type: specs.NetworkNamespace, Path: testNetNS}, + {Type: specs.IPCNamespace, Path: pins.IPC}, + {Type: specs.UTSNamespace, Path: pins.UTS}, + {Type: specs.PIDNamespace, Path: "/proc/42/ns/pid"}, + }, spec.Linux.Namespaces) + }) + + t.Run("host network keeps only the IPC namespace", func(t *testing.T) { + cfg := &Config{ID: testID, Hostname: "node", HostNetwork: true} + spec := buildSpec(cfg, Pins{IPC: pins.IPC}, nil, "") + assert.Equal(t, []specs.LinuxNamespace{{Type: specs.IPCNamespace, Path: pins.IPC}, {Type: specs.PIDNamespace}}, spec.Linux.Namespaces) + }) + + t.Run("host ipc keeps the network and UTS namespaces", func(t *testing.T) { + cfg := &Config{ID: testID, Hostname: "pod", HostIPC: true, NetNSPath: testNetNS} + spec := buildSpec(cfg, Pins{UTS: pins.UTS}, nil, "") + assert.Equal(t, []specs.LinuxNamespace{ + {Type: specs.NetworkNamespace, Path: testNetNS}, + {Type: specs.UTSNamespace, Path: pins.UTS}, + {Type: specs.PIDNamespace}, + }, spec.Linux.Namespaces) + }) + + t.Run("all host namespaces still carry a Linux section", func(t *testing.T) { + cfg := &Config{ID: testID, Hostname: "node", HostNetwork: true, HostIPC: true, HostPID: true} + spec := buildSpec(cfg, Pins{}, nil, "") + require.NotNil(t, spec.Linux, "CRI tells a processless sandbox by the Linux section") + assert.Empty(t, spec.Linux.Namespaces, "no PID namespace declaration for a host PID pod") + }) +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/doc.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/doc.go new file mode 100644 index 0000000000..50621796e8 --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/doc.go @@ -0,0 +1,105 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package sandbox implements the Sandbox API (runtime.sandbox.v1) for a pod +// that has no pause container. +// +// # Configuration +// +// The shim serves the io.containerd.runc.v2 runtime type from a different +// binary, so CRI treats the handler exactly as runc (runc options, features, +// cgroup driver); the handler names the binary and the sandboxer: +// +// [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] +// runtime_type = "io.containerd.runc.v2" +// runtime_path = "/usr/local/bin/containerd-shim-sandboxed-runc-v2" +// sandboxer = "shim" +// disable_pause_image_pull = true +// +// # What a pause sandbox does today +// +// With the podsandbox controller a pod sandbox is a container: CRI pulls the +// pause image, builds an OCI spec for it (hostname, namespaces, sysctls, +// cgroup, security options), creates the pod shared files under the CRI state +// directory (/sandboxes//{hostname,hosts,resolv.conf,shm}) and runs +// the pause process through the runc shim. The pause process exists only to +// keep the pod network, IPC and UTS namespaces alive: containers of the pod +// join /proc//ns/{net,ipc,uts} and CRI bind-mounts the shared +// files, which it created itself, into every container. +// +// # What this package does instead +// +// The shim is the sandbox. CreateSandbox receives the CRI PodSandboxConfig +// (as the sandbox options) and the network namespace CRI created and +// configured with CNI, and sets the pod up, without a process unless the pod +// shares its PID namespace: +// +// - Namespaces (namespaces.go). A locked OS thread enters the CRI network +// namespace, unshares an IPC and a UTS namespace, sets the pod hostname, +// writes the pod sysctls (net.* land in the pod network namespace, +// kernel.shm*, kernel.msg*, kernel.sem and fs.mqueue.* in the pod IPC +// namespace) and bind-mounts its /proc/self/task//ns/{ipc,uts} onto +// files under /ns. The thread is then discarded; the bind mounts, +// the pins, keep the namespaces alive the same way CRI keeps the network +// namespace alive under /var/run/netns. A namespace the pod shares with +// the host (hostNetwork, hostIPC) is neither created nor pinned. +// +// - Shared files (bundle.go). The shim writes hostname, copies the host +// hosts file, writes resolv.conf from the pod DNS config (or copies the +// host one) and mounts the pod shm tmpfs, all under its bundle instead of +// the CRI state directory, and lists them as Mounts of the sandbox spec. +// +// - Shared PID namespace (init.go). A pod that shares its PID namespace +// (shareProcessNamespace) needs a durable PID 1 for it, the role pause +// plays: the shim runs containerd-sandbox-init through runc, in the pod +// cgroup and the pod namespaces, and the spec names its PID namespace +// for the containers to join. Every other pod has no sandbox process. +// +// - Spec (buildSpec in config.go). StartSandbox returns pid 0 and a +// synthesized OCI spec that carries the namespace paths, the sysctls, the +// shared file mounts and the annotations of the request. It describes what +// the sandbox holds; there is no process it could describe. +// +// CRI consumes that spec where it used the pause pid before: a sandbox +// reporting pid 0 with a Linux section in its spec joins containers to the +// namespace paths of the spec (a namespace missing from the spec is the host +// one when the pod asked for it, and an error otherwise), and bind-mounts the +// shared files from the spec mount sources when nothing exists at the fixed +// CRI paths. Sandboxes with a pid, and older Sandbox API shims returning no +// spec, keep the pid derived paths unchanged. +// +// # Lifetime and cleanup +// +// One shim process serves one pod: its sandbox and, through the wrapped task +// service of containerd-shim-runc-v2 (task.go in package main), the runc tasks +// of its containers. StopSandbox releases what the sandbox holds and +// ShutdownSandbox ends the shim; CRI stops and removes the containers of a +// pod before either. Everything the shim creates lives under its bundle at +// fixed paths, so Cleanup needs no state and runs the same way on create +// failure, on stop and from "shim delete". +// +// # Not supported yet +// +// User namespaced pods need their namespaces created inside the +// pod user namespace, which a multithreaded shim cannot enter; they are +// rejected, as are user.* sysctls. The shim requires cgroup v2 and does not +// run rootless. SELinux labels are not applied yet: the pod label allocation +// moves into the CRI layer with +// https://github.com/containerd/containerd/pull/14108, after which the shim +// applies the labels it is handed (see the TODO in CreateSandbox). +package sandbox diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/init.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/init.go new file mode 100644 index 0000000000..5f2ad84ea9 --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/init.go @@ -0,0 +1,328 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "time" + + "github.com/containerd/errdefs" + gorunc "github.com/containerd/go-runc" + "github.com/containerd/log" + specs "github.com/opencontainers/runtime-spec/specs-go" + "golang.org/x/sys/unix" + + "github.com/containerd/containerd/v2/cmd/containerd-shim-runc-v2/process" + "github.com/containerd/containerd/v2/cmd/containerd-shim-runc-v2/runc" + "github.com/containerd/containerd/v2/contrib/seccomp" + "github.com/containerd/containerd/v2/core/containers" + "github.com/containerd/containerd/v2/pkg/namespaces" + "github.com/containerd/containerd/v2/pkg/oci" + "github.com/containerd/containerd/v2/pkg/sys/reaper" +) + +// InitBinary is the PID 1 program of a pod that shares its PID namespace, +// built from cmd/containerd-sandbox-init and installed next to the shim. +const InitBinary = "containerd-sandbox-init" + +const ( + // initPath is where the program is mounted inside its container. + initPath = "/" + InitBinary + // initRootfs is the rootfs of the init container, relative to the + // sandbox bundle: an empty tmpfs. + initRootfs = "rootfs" + // pidPin is the pin of the PID namespace the init is PID 1 of, beside + // the IPC and UTS pins. + pidPin = "pid" + // initUser is the user the init runs as: not root, and not nobody + // (65534), which workloads commonly run as. + initUser = 65535 + // initOOMScoreAdj is what CRI gives a pause container. + initOOMScoreAdj = -998 + // initStopTimeout bounds the wait for the init to be reaped after + // SIGKILL. + initStopTimeout = 10 * time.Second +) + +// initProcess is the containerd-sandbox-init runc container of a pod that +// shares its PID namespace (see the package doc). Its container id is the +// sandbox id, which is what "shim delete" force-deletes once the shim is +// gone. +type initProcess struct { + runtime *gorunc.Runc + id string + pid int + // pidNS is the pin of the PID namespace, the path the containers join. + pidNS string + // exited is closed once the exit of the process has been reaped; exit + // is valid from then on. + exited chan struct{} + exit gorunc.Exit +} + +// initBinary locates containerd-sandbox-init: next to the shim binary, then +// in PATH. +func initBinary() (string, error) { + if self, err := os.Executable(); err == nil { + p := filepath.Join(filepath.Dir(self), InitBinary) + if _, err := os.Stat(p); err == nil { + return p, nil + } + } + p, err := exec.LookPath(InitBinary) + if err != nil { + return "", fmt.Errorf("%s is not installed next to %s or in PATH: %w", InitBinary, BinaryName, errdefs.ErrFailedPrecondition) + } + return p, nil +} + +// cgroupsPath places the init container under the pod cgroup the way CRI +// places the containers of the pod: a systemd slice becomes runc's +// "slice:prefix:name", a cgroupfs path gets the container id appended. +func cgroupsPath(parent, id string) string { + if base := path.Base(parent); strings.HasSuffix(base, ".slice") { + return base + ":cri-containerd:" + id + } + return filepath.Join(parent, id) +} + +// initSpec is the OCI spec of the init container. The rootfs holds nothing +// but the program; the process has no capabilities, cannot gain privileges, +// runs under containerd's default seccomp profile and is scheduled and +// OOM-scored as a pause container is. Its /proc, that of the pod PID +// namespace, has the masked and read-only paths of a default container spec, +// as that of a pause container has. It joins the pod namespaces the sandbox +// holds and creates the PID namespace. The hostname is left alone: it is that +// of the joined UTS namespace. +// +// TODO: the pod SELinux label is not applied yet, as for the files the shim +// creates (see the SELinux TODO in CreateSandbox). +func initSpec(ctx context.Context, cfg *Config, pins Pins, binary string) (*specs.Spec, error) { + defaults, err := oci.GenerateSpec(ctx, nil, &containers.Container{ID: cfg.ID}) + if err != nil { + return nil, err + } + oomScoreAdj := initOOMScoreAdj + // The CPU shares of a pause container (WithDefaultSandboxShares). + shares := uint64(2) + s := &specs.Spec{ + Version: specs.Version, + Process: &specs.Process{ + User: specs.User{UID: initUser, GID: initUser}, + Args: []string{initPath}, + Cwd: "/", + Capabilities: &specs.LinuxCapabilities{}, + NoNewPrivileges: true, + OOMScoreAdj: &oomScoreAdj, + }, + Root: &specs.Root{Path: initRootfs, Readonly: true}, + Mounts: []specs.Mount{ + {Destination: "/proc", Type: "proc", Source: "proc", Options: []string{"nosuid", "noexec", "nodev"}}, + {Destination: "/dev", Type: "tmpfs", Source: "tmpfs", Options: []string{"nosuid", "strictatime", "mode=755", "size=65536k"}}, + {Destination: initPath, Type: "bind", Source: binary, Options: []string{"bind", "ro", "nosuid", "nodev"}}, + }, + Linux: &specs.Linux{ + Resources: &specs.LinuxResources{CPU: &specs.LinuxCPU{Shares: &shares}}, + Namespaces: append([]specs.LinuxNamespace{{Type: specs.PIDNamespace}, {Type: specs.MountNamespace}}, podNamespaces(cfg, pins)...), + MaskedPaths: defaults.Linux.MaskedPaths, + ReadonlyPaths: defaults.Linux.ReadonlyPaths, + }, + } + if cfg.CgroupParent != "" { + s.Linux.CgroupsPath = cgroupsPath(cfg.CgroupParent, cfg.ID) + } + // The profile depends on the capabilities of the process, set above. + s.Linux.Seccomp = seccomp.DefaultProfile(s) + return s, nil +} + +// createInit runs "runc create" for the init container in the sandbox +// bundle: the rootfs is a tmpfs mounted at /rootfs, so that the +// mount points runc creates in it leave nothing behind, and the spec is +// written as /config.json. From here on the process exists and is +// PID 1 of the new PID namespace, pinned at /ns/pid, waiting in runc +// for StartSandbox to exec the program. On error nothing is left running; +// the Cleanup the caller runs detaches the rootfs and the pin. +func createInit(ctx context.Context, bundle string, cfg *Config, pins Pins) (_ *initProcess, retErr error) { + ns, err := namespaces.NamespaceRequired(ctx) + if err != nil { + return nil, err + } + binary, err := initBinary() + if err != nil { + return nil, err + } + // The runc options of the runtime handler, persisted by "shim start": + // the same runc, root and cgroup driver as the containers of the pod. + opts, err := runc.ReadOptions(bundle) + if err != nil { + return nil, fmt.Errorf("failed to read the runc options: %w", err) + } + rootfs := filepath.Join(bundle, initRootfs) + if err := unix.Mount("tmpfs", rootfs, "tmpfs", unix.MS_NOSUID|unix.MS_NODEV, "mode=0755,size=1m"); err != nil { + return nil, fmt.Errorf("failed to mount the init rootfs at %s: %w", rootfs, err) + } + spec, err := initSpec(ctx, cfg, pins, binary) + if err != nil { + return nil, err + } + config, err := json.Marshal(spec) + if err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(bundle, "config.json"), config, 0o600); err != nil { + return nil, fmt.Errorf("failed to write the init spec: %w", err) + } + + p := &initProcess{ + runtime: process.NewRunc(opts.GetRoot(), bundle, ns, opts.GetBinaryName(), opts.GetSystemdCgroup()), + id: cfg.ID, + exited: make(chan struct{}), + } + // Subscribed before the process exists so that its exit cannot be + // missed; the watcher learns the pid once runc reports it. + ec := reaper.Default.Subscribe() + pidCh := make(chan int, 1) + go p.watch(ec, pidCh) + defer func() { + if retErr != nil { + close(pidCh) + // A failed create may have left runc state and a process. + if err := p.runtime.Delete(ctx, p.id, &gorunc.DeleteOpts{Force: true}); err != nil { + log.G(ctx).WithError(err).WithField("id", p.id).Debugf("failed to delete %s after a failed create", InitBinary) + } + } + }() + + nullIO, err := gorunc.NewNullIO() + if err != nil { + return nil, err + } + defer nullIO.Close() + pidFile := filepath.Join(bundle, process.InitPidFile) + if err := p.runtime.Create(ctx, p.id, bundle, &gorunc.CreateOpts{ + IO: nullIO, + PidFile: pidFile, + NoPivot: opts.GetNoPivotRoot(), + NoNewKeyring: opts.GetNoNewKeyring(), + }); err != nil { + return nil, fmt.Errorf("failed to create the %s container: %w", InitBinary, err) + } + if p.pid, err = gorunc.ReadPidFile(pidFile); err != nil { + return nil, fmt.Errorf("failed to read the %s pid: %w", InitBinary, err) + } + pidCh <- p.pid + // Pinned like the IPC and UTS namespaces, so that the path the + // containers join never names whatever process reuses the pid. + if err := os.MkdirAll(filepath.Join(bundle, pinDir), 0o700); err != nil { + return nil, fmt.Errorf("failed to create namespace pin directory: %w", err) + } + p.pidNS = filepath.Join(bundle, pinDir, pidPin) + if err := pin(fmt.Sprintf("/proc/%d/ns/pid", p.pid), p.pidNS); err != nil { + return nil, err + } + return p, nil +} + +// watch delivers the exit of the init through exited. It drains the reaper +// subscription throughout (a subscriber that falls behind slows every reap +// in the shim) and remembers the exits seen before the pid is known, since +// the process can be reaped between "runc create" and the pid file read. +func (p *initProcess) watch(ec chan gorunc.Exit, pidCh <-chan int) { + defer reaper.Default.Unsubscribe(ec) + pid := -1 + seen := map[int]gorunc.Exit{} + for { + select { + case id, ok := <-pidCh: + if !ok { + return + } + if e, ok := seen[id]; ok { + p.finish(e) + return + } + pid, seen = id, nil + case e, ok := <-ec: + if !ok { + return + } + switch { + case pid < 0: + seen[e.Pid] = e + case e.Pid == pid: + p.finish(e) + return + } + } + } +} + +func (p *initProcess) finish(e gorunc.Exit) { + p.exit = e + close(p.exited) +} + +// stop ends the init with SIGKILL, the one signal it does not ignore, waits +// for its exit to be reaped and deletes its runc container. A process that +// already exited on its own is only deleted; the exit is the same either +// way. The wait is bounded by initStopTimeout. +func (p *initProcess) stop(ctx context.Context) (gorunc.Exit, error) { + exit, err := p.kill(ctx) + if err != nil { + return exit, err + } + return exit, p.delete(ctx) +} + +// kill ends the init with SIGKILL, unless it has exited already, and waits +// for its exit, at most initStopTimeout. +func (p *initProcess) kill(ctx context.Context) (gorunc.Exit, error) { + select { + case <-p.exited: + default: + if err := p.runtime.Kill(ctx, p.id, int(unix.SIGKILL), nil); err != nil { + // The process may have exited in the meantime; the wait tells. + log.G(ctx).WithError(err).WithField("id", p.id).Debugf("failed to kill %s", InitBinary) + } + select { + case <-p.exited: + case <-time.After(initStopTimeout): + return gorunc.Exit{}, fmt.Errorf("%s (pid %d) did not exit after SIGKILL", InitBinary, p.pid) + case <-ctx.Done(): + return gorunc.Exit{}, ctx.Err() + } + } + return p.exit, nil +} + +// delete removes the runc container of the init. Deleting it again is fine. +func (p *initProcess) delete(ctx context.Context) error { + if err := p.runtime.Delete(ctx, p.id, &gorunc.DeleteOpts{Force: true}); err != nil { + return fmt.Errorf("failed to delete the %s container: %w", InitBinary, err) + } + return nil +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/init_test.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/init_test.go new file mode 100644 index 0000000000..1e314bfdef --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/init_test.go @@ -0,0 +1,202 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "fmt" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "testing" + "time" + + gorunc "github.com/containerd/go-runc" + "github.com/moby/sys/mountinfo" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" + + "github.com/containerd/containerd/v2/contrib/seccomp" + "github.com/containerd/containerd/v2/core/containers" + "github.com/containerd/containerd/v2/pkg/namespaces" + "github.com/containerd/containerd/v2/pkg/oci" + "github.com/containerd/containerd/v2/pkg/sys/reaper" + "github.com/containerd/containerd/v2/pkg/testutil" +) + +func TestCgroupsPath(t *testing.T) { + assert.Equal(t, "/kubepods/burstable/pod1/"+testID, cgroupsPath("/kubepods/burstable/pod1", testID)) + assert.Equal(t, "kubepods-burstable-pod1.slice:cri-containerd:"+testID, cgroupsPath("/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod1.slice", testID)) + assert.Equal(t, "test.slice:cri-containerd:"+testID, cgroupsPath("/test.slice", testID)) +} + +func TestInitSpec(t *testing.T) { + ctx := namespaces.WithNamespace(context.Background(), "testing") + cfg := &Config{ID: testID, NetNSPath: testNetNS, CgroupParent: "/kubepods/pod1", SharedPID: true} + pins := Pins{IPC: "/bundle/ns/ipc", UTS: "/bundle/ns/uts"} + binary := "/usr/local/bin/" + InitBinary + spec, err := initSpec(ctx, cfg, pins, binary) + require.NoError(t, err) + + require.NotNil(t, spec.Process) + assert.Equal(t, []string{"/" + InitBinary}, spec.Process.Args) + assert.Equal(t, specs.User{UID: initUser, GID: initUser}, spec.Process.User) + assert.True(t, spec.Process.NoNewPrivileges) + assert.Equal(t, &specs.LinuxCapabilities{}, spec.Process.Capabilities, "no capabilities") + require.NotNil(t, spec.Process.OOMScoreAdj) + assert.Equal(t, initOOMScoreAdj, *spec.Process.OOMScoreAdj) + assert.Equal(t, &specs.Root{Path: "rootfs", Readonly: true}, spec.Root) + assert.Equal(t, specs.Mount{Destination: "/" + InitBinary, Type: "bind", Source: binary, Options: []string{"bind", "ro", "nosuid", "nodev"}}, spec.Mounts[len(spec.Mounts)-1]) + assert.Equal(t, []specs.LinuxNamespace{ + {Type: specs.PIDNamespace}, + {Type: specs.MountNamespace}, + {Type: specs.NetworkNamespace, Path: testNetNS}, + {Type: specs.IPCNamespace, Path: pins.IPC}, + {Type: specs.UTSNamespace, Path: pins.UTS}, + }, spec.Linux.Namespaces) + assert.Equal(t, "/kubepods/pod1/"+testID, spec.Linux.CgroupsPath) + assert.Empty(t, spec.Hostname, "the hostname of the joined UTS namespace is left alone") + + defaults, err := oci.GenerateSpec(ctx, nil, &containers.Container{ID: testID}) + require.NoError(t, err) + assert.Equal(t, defaults.Linux.MaskedPaths, spec.Linux.MaskedPaths) + assert.Equal(t, defaults.Linux.ReadonlyPaths, spec.Linux.ReadonlyPaths) + assert.Contains(t, spec.Linux.ReadonlyPaths, "/proc/sys") + assert.Equal(t, seccomp.DefaultProfile(spec), spec.Linux.Seccomp) + + t.Run("host namespaces", func(t *testing.T) { + spec, err := initSpec(ctx, &Config{ID: testID, HostNetwork: true, HostIPC: true, SharedPID: true}, Pins{}, binary) + require.NoError(t, err) + assert.Equal(t, []specs.LinuxNamespace{{Type: specs.PIDNamespace}, {Type: specs.MountNamespace}}, spec.Linux.Namespaces) + assert.Empty(t, spec.Linux.CgroupsPath) + }) +} + +// startReaper makes the test process reap its children the way the shim +// does; the runc commands and the exit tracking of the init rely on it. +func startReaper(t *testing.T) { + require.NoError(t, reaper.SetSubreaper(1)) + monitor := gorunc.Monitor + gorunc.Monitor = reaper.Default + ch := make(chan os.Signal, 32) + signal.Notify(ch, unix.SIGCHLD) + done := make(chan struct{}) + go func() { + for { + select { + case <-ch: + _ = reaper.Reap() + case <-done: + return + } + } + }() + t.Cleanup(func() { + signal.Stop(ch) + close(done) + gorunc.Monitor = monitor + _ = reaper.SetSubreaper(0) + }) +} + +// newInit creates the init of a pod in the host network and IPC namespaces +// in a fresh sandbox bundle, stopped and cleaned up with the test. The test +// needs root, runc and the installed init binary. +func newInit(t *testing.T) (context.Context, string, *initProcess) { + testutil.RequiresRoot(t) + if _, err := exec.LookPath("runc"); err != nil { + t.Skip("runc is not installed") + } + if _, err := initBinary(); err != nil { + t.Skip(err.Error()) + } + startReaper(t) + + ctx := namespaces.WithNamespace(context.Background(), "testing") + bundle := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(bundle, initRootfs), 0o711)) + t.Cleanup(func() { _ = Cleanup(ctx, bundle) }) + cfg := &Config{ID: fmt.Sprintf("sandbox-init-test-%d", os.Getpid()), HostNetwork: true, HostIPC: true, SharedPID: true, CgroupParent: "/containerd-test"} + p, err := createInit(ctx, bundle, cfg, Pins{}) + require.NoError(t, err) + t.Cleanup(func() { _, _ = p.stop(ctx) }) + return ctx, bundle, p +} + +// startInit execs the program of p and waits for it to run. +func startInit(ctx context.Context, t *testing.T, p *initProcess) { + require.NoError(t, p.runtime.Start(ctx, p.id)) + require.Eventually(t, func() bool { return running(p.pid) }, 5*time.Second, 20*time.Millisecond) +} + +// running reports whether the init program is running as pid: the comm of +// the process is the binary name, truncated by the kernel. +func running(pid int) bool { + comm, err := os.ReadFile(fmt.Sprintf("/proc/%d/comm", pid)) + return err == nil && strings.HasPrefix(InitBinary, strings.TrimSpace(string(comm))) +} + +func TestInitLifecycle(t *testing.T) { + ctx, bundle, p := newInit(t) + assert.Greater(t, p.pid, 0) + mounted, err := mountinfo.Mounted(p.pidNS) + require.NoError(t, err) + assert.True(t, mounted, "the PID namespace is pinned") + assert.False(t, running(p.pid), "created, not started") + + startInit(ctx, t, p) + + exit, err := p.stop(ctx) + require.NoError(t, err) + assert.Equal(t, 128+int(unix.SIGKILL), exit.Status) + assert.NoFileExists(t, fmt.Sprintf("/proc/%d", p.pid)) + + // Stopping again deletes what is already deleted and reports the same exit. + again, err := p.stop(ctx) + require.NoError(t, err) + assert.Equal(t, exit, again) + + require.NoError(t, Cleanup(ctx, bundle)) + for _, p := range []string{filepath.Join(bundle, initRootfs), p.pidNS} { + mounted, err := mountinfo.Mounted(p) + if err == nil { + assert.False(t, mounted, "%s is detached", p) + } + } +} + +func TestInitExit(t *testing.T) { + ctx, _, p := newInit(t) + startInit(ctx, t, p) + + // An init killed from outside the shim is noticed and its exit kept. + require.NoError(t, unix.Kill(p.pid, unix.SIGKILL)) + select { + case <-p.exited: + case <-time.After(5 * time.Second): + t.Fatal("the exit of the init was not seen") + } + exit, err := p.stop(ctx) + require.NoError(t, err) + assert.Equal(t, 128+int(unix.SIGKILL), exit.Status) +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/namespaces.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/namespaces.go new file mode 100644 index 0000000000..4fb8936075 --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/namespaces.go @@ -0,0 +1,233 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "errors" + "fmt" + "io/fs" + "maps" + "os" + "path/filepath" + "runtime" + "slices" + + "golang.org/x/sys/unix" +) + +// Pod namespaces without a process; see the package doc. The approach follows +// nerdbox's shared resources implementation (github.com/containerd/nerdbox, +// internal/vminit/sharedresources). + +const ( + // pinDir is the bundle directory holding the namespace pins. + pinDir = "ns" + ipcPin = "ipc" + utsPin = "uts" +) + +// Pins are the bind-mounted namespace files a sandbox holds. An empty path +// means the pod uses the host namespace of that type. +type Pins struct { + IPC string + UTS string +} + +// setupNamespaces creates and pins the pod namespaces under the bundle and +// applies the hostname and sysctls to them. On error the caller runs Cleanup. +func setupNamespaces(bundle string, cfg *Config) (Pins, error) { + ipc, uts := !cfg.HostIPC, !cfg.HostNetwork + if !ipc && !uts { + // Nothing to hold, and the sysctl policy leaves nothing to apply to + // namespaces the pod shares with the host. + return Pins{}, nil + } + dir := filepath.Join(bundle, pinDir) + if err := os.Mkdir(dir, 0o700); err != nil && !errors.Is(err, fs.ErrExist) { + return Pins{}, fmt.Errorf("failed to create namespace pin directory: %w", err) + } + + var pins Pins + errCh := make(chan error, 1) + go func() { + // The thread is locked while it is in the pod namespaces and put + // back into the host namespaces before it is released, whether the + // work succeeded or not: a thread Go cannot discard (the main thread) + // must never stay in the pod. The pins keep the namespaces alive. + runtime.LockOSThread() + host, err := openThreadNamespaces() + if err != nil { + errCh <- err + return + } + err = func() error { + if cfg.NetNSPath != "" { + // Entered first so that net sysctls apply to the pod network + // namespace. + if err := setns(cfg.NetNSPath, unix.CLONE_NEWNET); err != nil { + return err + } + } + var flags int + if ipc { + flags |= unix.CLONE_NEWIPC + } + if uts { + flags |= unix.CLONE_NEWUTS + } + if err := unix.Unshare(flags); err != nil { + return fmt.Errorf("failed to create pod namespaces: %w", err) + } + if uts { + if err := unix.Sethostname([]byte(cfg.Hostname)); err != nil { + return fmt.Errorf("failed to set hostname %q: %w", cfg.Hostname, err) + } + } + for _, k := range slices.Sorted(maps.Keys(cfg.Sysctls)) { + if err := writeSysctl(k, cfg.Sysctls[k]); err != nil { + return err + } + } + tid := unix.Gettid() + if ipc { + p := filepath.Join(dir, ipcPin) + if err := pin(threadNamespacePath(tid, "ipc"), p); err != nil { + return err + } + pins.IPC = p + } + if uts { + p := filepath.Join(dir, utsPin) + if err := pin(threadNamespacePath(tid, "uts"), p); err != nil { + return err + } + pins.UTS = p + } + return nil + }() + if rerr := host.restore(); rerr != nil { + // The thread stays locked and is discarded with the goroutine. + errCh <- errors.Join(err, rerr) + return + } + runtime.UnlockOSThread() + errCh <- err + }() + if err := <-errCh; err != nil { + return Pins{}, err + } + return pins, nil +} + +// threadNamespaces holds the namespace files of a thread before it moves, +// so that it can be moved back. +type threadNamespaces struct { + net, ipc, uts int +} + +func openThreadNamespaces() (*threadNamespaces, error) { + t := &threadNamespaces{net: -1, ipc: -1, uts: -1} + for _, ns := range []struct { + name string + fd *int + }{{"net", &t.net}, {"ipc", &t.ipc}, {"uts", &t.uts}} { + fd, err := unix.Open("/proc/thread-self/ns/"+ns.name, unix.O_RDONLY|unix.O_CLOEXEC, 0) + if err != nil { + t.close() + return nil, fmt.Errorf("failed to open the %s namespace of the thread: %w", ns.name, err) + } + *ns.fd = fd + } + return t, nil +} + +// restore moves the thread back into its original namespaces and closes them. +func (t *threadNamespaces) restore() error { + defer t.close() + var errs []error + for _, ns := range []struct { + fd int + flag int + which string + }{{t.uts, unix.CLONE_NEWUTS, "uts"}, {t.ipc, unix.CLONE_NEWIPC, "ipc"}, {t.net, unix.CLONE_NEWNET, "net"}} { + if err := unix.Setns(ns.fd, ns.flag); err != nil { + errs = append(errs, fmt.Errorf("failed to move the thread back into the host %s namespace: %w", ns.which, err)) + } + } + return errors.Join(errs...) +} + +func (t *threadNamespaces) close() { + for _, fd := range []int{t.net, t.ipc, t.uts} { + if fd >= 0 { + unix.Close(fd) + } + } + t.net, t.ipc, t.uts = -1, -1, -1 +} + +// setns moves the calling thread into the namespace pinned at path. +func setns(path string, nstype int) error { + fd, err := unix.Open(path, unix.O_RDONLY|unix.O_CLOEXEC, 0) + if err != nil { + return fmt.Errorf("failed to open namespace %s: %w", path, err) + } + defer unix.Close(fd) + if err := unix.Setns(fd, nstype); err != nil { + return fmt.Errorf("failed to enter namespace %s: %w", path, err) + } + return nil +} + +// threadNamespacePath is the namespace file of one thread. /proc/self/ns +// would name the namespaces of the main thread, not of the thread that +// changed them. +func threadNamespacePath(tid int, nsType string) string { + return fmt.Sprintf("/proc/%d/task/%d/ns/%s", os.Getpid(), tid, nsType) +} + +// pin bind-mounts the namespace file src onto a new empty file at target so +// that the namespace persists without a process in it. +func pin(src, target string) error { + f, err := os.OpenFile(target, os.O_RDONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return fmt.Errorf("failed to create namespace pin %s: %w", target, err) + } + f.Close() + if err := unix.Mount(src, target, "none", unix.MS_BIND, ""); err != nil { + os.Remove(target) + return fmt.Errorf("failed to pin namespace %s at %s: %w", src, target, err) + } + return nil +} + +// writeSysctl sets a sysctl for the namespaces of the calling thread: the +// files under /proc/sys/net resolve against the network namespace of the +// caller and the IPC ones against its IPC namespace, so this must run on the +// thread that entered the pod namespaces. +func writeSysctl(key, value string) error { + p, _, err := sysctlPath(key) + if err != nil { + return err + } + if err := os.WriteFile(p, []byte(value), 0o644); err != nil { + return fmt.Errorf("failed to set sysctl %s=%q: %w", key, value, err) + } + return nil +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/namespaces_test.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/namespaces_test.go new file mode 100644 index 0000000000..4f78ff665a --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/namespaces_test.go @@ -0,0 +1,152 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/moby/sys/mountinfo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" + + "github.com/containerd/containerd/v2/pkg/testutil" +) + +// inNamespace runs fn on a thread that entered the namespace pinned at pin +// and returns what it reads. The thread is locked and discarded afterwards, +// like the one that created the namespace. +func inNamespace(t *testing.T, pin string, nsType int, fn func() (string, error)) string { + t.Helper() + var ( + result string + err error + ) + done := make(chan struct{}) + go func() { + defer close(done) + runtime.LockOSThread() + var fd int + if fd, err = unix.Open(pin, unix.O_RDONLY|unix.O_CLOEXEC, 0); err != nil { + return + } + defer unix.Close(fd) + if err = unix.Setns(fd, nsType); err != nil { + return + } + result, err = fn() + }() + <-done + require.NoError(t, err, "in namespace %s", pin) + return result +} + +func inode(t *testing.T, path string) uint64 { + t.Helper() + var st unix.Stat_t + require.NoError(t, unix.Stat(path, &st)) + return st.Ino +} + +func assertNoMountsUnder(t *testing.T, dir string) { + t.Helper() + mounts, err := mountinfo.GetMounts(mountinfo.PrefixFilter(dir)) + require.NoError(t, err) + assert.Empty(t, mounts, "no mounts left under %s", dir) +} + +func TestSetupNamespaces(t *testing.T) { + testutil.RequiresRoot(t) + ctx := context.Background() + bundle := t.TempDir() + t.Cleanup(func() { _ = Cleanup(ctx, bundle) }) + + pins, err := setupNamespaces(bundle, &Config{ + Hostname: "sandbox-test", + Sysctls: map[string]string{"kernel.shm_rmid_forced": "1"}, + }) + require.NoError(t, err) + assert.Equal(t, filepath.Join(bundle, pinDir, ipcPin), pins.IPC) + assert.Equal(t, filepath.Join(bundle, pinDir, utsPin), pins.UTS) + + // The pins are namespace files distinct from the host namespaces. + for _, tc := range []struct{ pin, nsType string }{{pins.IPC, "ipc"}, {pins.UTS, "uts"}} { + var st unix.Statfs_t + require.NoError(t, unix.Statfs(tc.pin, &st)) + assert.EqualValues(t, unix.NSFS_MAGIC, st.Type, "%s pin is an nsfs mount", tc.nsType) + assert.NotEqual(t, inode(t, "/proc/self/ns/"+tc.nsType), inode(t, tc.pin), "%s namespace is not the host one", tc.nsType) + } + + // The pod UTS namespace carries the pod hostname; the host keeps its own. + hostHostname, err := os.Hostname() + require.NoError(t, err) + podHostname := inNamespace(t, pins.UTS, unix.CLONE_NEWUTS, func() (string, error) { + var u unix.Utsname + if err := unix.Uname(&u); err != nil { + return "", err + } + return unix.ByteSliceToString(u.Nodename[:]), nil + }) + assert.Equal(t, "sandbox-test", podHostname) + current, err := os.Hostname() + require.NoError(t, err) + assert.Equal(t, hostHostname, current) + + // The IPC sysctl was applied inside the pod IPC namespace. + value := inNamespace(t, pins.IPC, unix.CLONE_NEWIPC, func() (string, error) { + b, err := os.ReadFile("/proc/sys/kernel/shm_rmid_forced") + return strings.TrimSpace(string(b)), err + }) + assert.Equal(t, "1", value) + + // Cleanup detaches the pins and removes them; a second run is a no-op. + require.NoError(t, Cleanup(ctx, bundle)) + for _, p := range []string{pins.IPC, pins.UTS, filepath.Join(bundle, pinDir)} { + _, err := os.Stat(p) + assert.True(t, os.IsNotExist(err), "%s should be gone", p) + } + assertNoMountsUnder(t, bundle) + require.NoError(t, Cleanup(ctx, bundle)) +} + +func TestSetupNamespacesHostModes(t *testing.T) { + testutil.RequiresRoot(t) + bundle := t.TempDir() + + // A pod in the host network and IPC namespaces holds nothing. + pins, err := setupNamespaces(bundle, &Config{HostNetwork: true, HostIPC: true}) + require.NoError(t, err) + assert.Equal(t, Pins{}, pins) + _, err = os.Stat(filepath.Join(bundle, pinDir)) + assert.True(t, os.IsNotExist(err), "no pin directory without pins") + + // Host network with a pod IPC namespace: only the IPC pin exists. + pins, err = setupNamespaces(bundle, &Config{HostNetwork: true}) + require.NoError(t, err) + t.Cleanup(func() { _ = Cleanup(context.Background(), bundle) }) + assert.NotEmpty(t, pins.IPC) + assert.Empty(t, pins.UTS) + _, err = os.Stat(filepath.Join(bundle, pinDir, utsPin)) + assert.True(t, os.IsNotExist(err)) +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service.go new file mode 100644 index 0000000000..f2d5ca2cca --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service.go @@ -0,0 +1,447 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "fmt" + "strconv" + "sync" + "time" + + cgroupsv2 "github.com/containerd/cgroups/v3/cgroup2" + api "github.com/containerd/containerd/api/runtime/sandbox/v1" + "github.com/containerd/containerd/api/types" + "github.com/containerd/errdefs" + "github.com/containerd/errdefs/pkg/errgrpc" + "github.com/containerd/log" + "github.com/containerd/platforms" + "github.com/containerd/plugin" + "github.com/containerd/plugin/registry" + "github.com/containerd/ttrpc" + "github.com/containerd/typeurl/v2" + "github.com/moby/sys/userns" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "google.golang.org/protobuf/types/known/anypb" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/containerd/containerd/v2/pkg/protobuf" + "github.com/containerd/containerd/v2/pkg/shutdown" + "github.com/containerd/containerd/v2/plugins" +) + +// BinaryName is the name of this shim, used in messages. It serves the +// io.containerd.runc.v2 runtime type; a runtime handler reaches it through +// runtime_path. +const BinaryName = "containerd-shim-sandboxed-runc-v2" + +func init() { + registry.Register(&plugin.Registration{ + Type: plugins.TTRPCPlugin, + ID: "sandbox", + Requires: []plugin.Type{ + plugins.InternalPlugin, + }, + InitFn: func(ic *plugin.InitContext) (any, error) { + ss, err := ic.GetByID(plugins.InternalPlugin, "shutdown") + if err != nil { + return nil, err + } + return NewService(ss.(shutdown.Service)), nil + }, + }) +} + +// Service implements the sandbox ttrpc service. containerd starts one shim per +// sandbox, so a Service hosts at most one. +type Service struct { + shutdown shutdown.Service + + mu sync.Mutex + sandbox *instance +} + +type instance struct { + cfg *Config + bundle string + pins Pins + spec *anypb.Any + createdAt time.Time + // init is PID 1 of the pod PID namespace, nil unless the pod shares it. + init *initProcess + + started bool + stopped bool + exitStatus uint32 + exitedAt time.Time + // done is closed once the sandbox is stopped. + done chan struct{} + // released is set once the init container, the pins and the files are + // gone; a stopped sandbox may still hold them if releasing failed. + released bool +} + +var _ api.TTRPCSandboxService = (*Service)(nil) + +// NewService returns a sandbox service. sd ends the shim on ShutdownSandbox. +func NewService(sd shutdown.Service) *Service { + return &Service{shutdown: sd} +} + +// RegisterTTRPC implements shim.TTRPCService. +func (s *Service) RegisterTTRPC(server *ttrpc.Server) error { + api.RegisterTTRPCSandboxService(server, s) + return nil +} + +// findSandbox returns the hosted sandbox if it is the one asked for. The +// caller holds s.mu. +func (s *Service) findSandbox(id string) (*instance, error) { + if s.sandbox == nil || s.sandbox.cfg.ID != id { + return nil, fmt.Errorf("sandbox %q: %w", id, errdefs.ErrNotFound) + } + return s.sandbox, nil +} + +// CreateSandbox sets the pod up: namespaces, hostname, sysctls and shared +// files. Nothing is left behind on failure. +func (s *Service) CreateSandbox(ctx context.Context, r *api.CreateSandboxRequest) (_ *api.CreateSandboxResponse, retErr error) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.sandbox != nil { + return nil, errgrpc.ToGRPCf(errdefs.ErrAlreadyExists, "shim already hosts sandbox %q", s.sandbox.cfg.ID) + } + if userns.RunningInUserNS() { + return nil, errgrpc.ToGRPCf(errdefs.ErrNotImplemented, "%s does not support rootless containerd", BinaryName) + } + bundle := r.GetBundlePath() + if bundle == "" { + return nil, errgrpc.ToGRPCf(errdefs.ErrInvalidArgument, "sandbox %q: no bundle path", r.GetSandboxID()) + } + cfg, err := ConfigFromRequest(r) + if err != nil { + return nil, errgrpc.ToGRPC(err) + } + log.G(ctx).WithField("id", cfg.ID).Warnf("%s is experimental: the pod sandbox runs without a pause container", BinaryName) + + defer func() { + if retErr != nil { + if err := Cleanup(ctx, bundle); err != nil { + log.G(ctx).WithError(err).WithField("id", cfg.ID).Warn("failed to clean up after a failed sandbox creation") + } + } + }() + + pins, err := setupNamespaces(bundle, cfg) + if err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to set up the pod namespaces") + } + // TODO: SELinux. The shim allocates no pod process label and CRI does not + // allocate one for a Sandbox API shim yet, so on an enforcing host the + // containers of the pod without explicit SELinux options get independent + // MCS labels and relabel the shared files and /dev/shm in turn, and the + // pod-level SELinux options are not applied. Waits for + // https://github.com/containerd/containerd/pull/14108, which allocates the + // pod label in the CRI layer; the shim then applies the labels it is + // handed to the files it creates. + mounts, err := setupFiles(bundle, cfg) + if err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to set up the pod files") + } + var ( + initP *initProcess + pidNS string + ) + if cfg.SharedPID { + // Created last, once the pins it joins exist; if anything below + // fails it is stopped before the deferred Cleanup runs. + if initP, err = createInit(ctx, bundle, cfg, pins); err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to create the pod PID namespace") + } + defer func() { + if retErr != nil { + if _, err := initP.stop(ctx); err != nil { + log.G(ctx).WithError(err).WithField("id", cfg.ID).Warnf("failed to stop %s after a failed sandbox creation", InitBinary) + } + } + }() + pidNS = initP.pidNS + } + spec, err := typeurl.MarshalAnyToProto(buildSpec(cfg, pins, mounts, pidNS)) + if err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to marshal the sandbox spec") + } + + inst := &instance{ + cfg: cfg, + bundle: bundle, + pins: pins, + spec: spec, + createdAt: time.Now(), + init: initP, + done: make(chan struct{}), + } + s.sandbox = inst + if initP != nil { + // The watcher outlives this request: it keeps the namespace and + // log fields of the request without its cancellation. + go s.watchInit(context.WithoutCancel(ctx), inst) + } + return &api.CreateSandboxResponse{}, nil +} + +// watchInit ends the sandbox when its init exits on its own: the kernel +// kills every process in a PID namespace whose PID 1 is gone, so the pod is +// over, as it is when a pause container dies. StopSandbox, which kills the +// init itself, records the exit first and leaves nothing to do here. +func (s *Service) watchInit(ctx context.Context, inst *instance) { + <-inst.init.exited + s.mu.Lock() + defer s.mu.Unlock() + if inst.stopped { + return + } + log.G(ctx).WithField("id", inst.cfg.ID).Warnf("%s exited with status %d, stopping the sandbox", InitBinary, inst.init.exit.Status) + if err := s.stop(ctx, inst); err != nil { + log.G(ctx).WithError(err).WithField("id", inst.cfg.ID).Error("failed to stop the sandbox after its init exited") + } +} + +// StartSandbox marks the sandbox running. There is no process to start +// unless the pod shares its PID namespace, in which case the program of its +// PID 1 is exec'd now. The pid reported is 0 either way: the sandbox is +// path-backed and the spec describes what it holds, that PID namespace +// included. Starting it again returns the same response. +func (s *Service) StartSandbox(ctx context.Context, r *api.StartSandboxRequest) (*api.StartSandboxResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + inst, err := s.findSandbox(r.GetSandboxID()) + if err != nil { + return nil, errgrpc.ToGRPC(err) + } + if inst.stopped { + return nil, errgrpc.ToGRPCf(errdefs.ErrFailedPrecondition, "sandbox %q is stopped", inst.cfg.ID) + } + if !inst.started && inst.init != nil { + if err := inst.init.runtime.Start(ctx, inst.init.id); err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to start %s", InitBinary) + } + } + inst.started = true + return &api.StartSandboxResponse{ + Pid: 0, + CreatedAt: protobuf.ToTimestamp(inst.createdAt), + Spec: inst.spec, + }, nil +} + +// Platform implements the sandbox service: the sandbox runs on the host. +func (s *Service) Platform(ctx context.Context, r *api.PlatformRequest) (*api.PlatformResponse, error) { + return &api.PlatformResponse{ + Platform: types.OCIPlatformToProto([]ocispec.Platform{platforms.DefaultSpec()})[0], + }, nil +} + +// StopSandbox releases the pod namespaces and files, and kills the init of a +// shared-PID pod, which ends whatever is left in the pod PID namespace. +// Containers still running keep the other namespaces they joined until they +// exit; CRI stops them first. Stopping twice is fine. +func (s *Service) StopSandbox(ctx context.Context, r *api.StopSandboxRequest) (*api.StopSandboxResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + inst, err := s.findSandbox(r.GetSandboxID()) + if err != nil { + return nil, errgrpc.ToGRPC(err) + } + if err := s.stop(ctx, inst); err != nil { + return nil, errgrpc.ToGRPC(err) + } + return &api.StopSandboxResponse{}, nil +} + +// stop marks the sandbox stopped and releases what it holds. The init of a +// shared-PID pod is killed first, and its exit becomes the exit of the +// sandbox. Once the init is gone the sandbox is stopped, and Status and Wait +// report it, even if the release then fails: the pod PID namespace died with +// its init. A release that failed is retried by the next stop. The caller +// holds s.mu. +func (s *Service) stop(ctx context.Context, inst *instance) error { + if !inst.stopped { + exitedAt := time.Now() + if inst.init != nil { + exit, err := inst.init.kill(ctx) + if err != nil { + return fmt.Errorf("failed to stop sandbox %q: %w", inst.cfg.ID, err) + } + inst.exitStatus, exitedAt = uint32(exit.Status), exit.Timestamp + } + inst.stopped, inst.exitedAt = true, exitedAt + close(inst.done) + } + if inst.released { + return nil + } + if inst.init != nil { + if err := inst.init.delete(ctx); err != nil { + return fmt.Errorf("failed to stop sandbox %q: %w", inst.cfg.ID, err) + } + } + if err := Cleanup(ctx, inst.bundle); err != nil { + return fmt.Errorf("failed to clean up sandbox %q: %w", inst.cfg.ID, err) + } + inst.released = true + return nil +} + +// WaitSandbox blocks until the sandbox is stopped. The exit status is that +// of the init of a shared-PID pod, 0 for a sandbox without a process. +func (s *Service) WaitSandbox(ctx context.Context, r *api.WaitSandboxRequest) (*api.WaitSandboxResponse, error) { + s.mu.Lock() + inst, err := s.findSandbox(r.GetSandboxID()) + s.mu.Unlock() + if err != nil { + return nil, errgrpc.ToGRPC(err) + } + select { + case <-inst.done: + case <-ctx.Done(): + return nil, errgrpc.ToGRPC(ctx.Err()) + } + return &api.WaitSandboxResponse{ + ExitStatus: inst.exitStatus, + ExitedAt: protobuf.ToTimestamp(inst.exitedAt), + }, nil +} + +// SandboxStatus reports the lifecycle state: ready from StartSandbox until the +// sandbox is stopped, not ready before and after. The verbose info carries +// only shim specific keys; the CRI server builds its own verbose document. +func (s *Service) SandboxStatus(ctx context.Context, r *api.SandboxStatusRequest) (*api.SandboxStatusResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + inst, err := s.findSandbox(r.GetSandboxID()) + if err != nil { + return nil, errgrpc.ToGRPC(err) + } + resp := &api.SandboxStatusResponse{ + SandboxID: inst.cfg.ID, + Pid: 0, + State: runtime.PodSandboxState_SANDBOX_NOTREADY.String(), + CreatedAt: protobuf.ToTimestamp(inst.createdAt), + } + switch { + case inst.stopped: + resp.ExitedAt = protobuf.ToTimestamp(inst.exitedAt) + case inst.started: + resp.State = runtime.PodSandboxState_SANDBOX_READY.String() + } + if r.GetVerbose() { + var initPid string + if inst.init != nil { + initPid = strconv.Itoa(inst.init.pid) + } + resp.Info = make(map[string]string, 5) + for _, kv := range [][2]string{ + {BinaryName + "/netns", inst.cfg.NetNSPath}, + {BinaryName + "/ipcns", inst.pins.IPC}, + {BinaryName + "/utsns", inst.pins.UTS}, + {BinaryName + "/hostname", inst.cfg.Hostname}, + {BinaryName + "/init-pid", initPid}, + } { + if kv[1] != "" { + resp.Info[kv[0]] = kv[1] + } + } + } + return resp, nil +} + +// PingSandbox implements the sandbox service. +func (s *Service) PingSandbox(ctx context.Context, r *api.PingRequest) (*api.PingResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + if _, err := s.findSandbox(r.GetSandboxID()); err != nil { + return nil, errgrpc.ToGRPC(err) + } + return &api.PingResponse{}, nil +} + +// ShutdownSandbox stops the sandbox if needed, forgets it and ends the shim. +// The containers of the pod are gone by then: CRI removes them before it +// removes the pod. Shutting down twice, or a shim that hosts no sandbox (its +// creation failed), is fine; a request for another sandbox than the hosted +// one is NotFound and leaves it running. +func (s *Service) ShutdownSandbox(ctx context.Context, r *api.ShutdownSandboxRequest) (*api.ShutdownSandboxResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.sandbox != nil { + inst, err := s.findSandbox(r.GetSandboxID()) + if err != nil { + return nil, errgrpc.ToGRPC(err) + } + if err := s.stop(ctx, inst); err != nil { + return nil, errgrpc.ToGRPC(err) + } + s.sandbox = nil + } + s.shutdown.Shutdown() + return &api.ShutdownSandboxResponse{}, nil +} + +// SandboxMetrics reports the cgroup v2 stats of the pod cgroup, encoded the +// way CRI expects sandbox metrics (a *cgroup2/stats.Metrics). The sandbox has +// no process of its own (the init of a shared-PID pod sits in the pod cgroup +// like any container), so the aggregate pod cgroup is the only meaningful +// source. CRI polls this every second; the cgroup files are read outside the +// lock. +func (s *Service) SandboxMetrics(ctx context.Context, r *api.SandboxMetricsRequest) (*api.SandboxMetricsResponse, error) { + s.mu.Lock() + inst, err := s.findSandbox(r.GetSandboxID()) + s.mu.Unlock() + if err != nil { + return nil, errgrpc.ToGRPC(err) + } + id, cgroupParent := inst.cfg.ID, inst.cfg.CgroupParent + if cgroupParent == "" { + return nil, errgrpc.ToGRPCf(errdefs.ErrNotFound, "sandbox %q has no cgroup parent", id) + } + cg, err := cgroupsv2.Load(cgroupParent) + if err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to load sandbox cgroup %q", cgroupParent) + } + stats, err := cg.StatFiltered(cgroupsv2.StatCPU | cgroupsv2.StatMemory) + if err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to read sandbox cgroup %q", cgroupParent) + } + data, err := typeurl.MarshalAnyToProto(stats) + if err != nil { + return nil, errgrpc.ToGRPCf(err, "failed to marshal sandbox metrics") + } + return &api.SandboxMetricsResponse{Metrics: &types.Metric{ + Timestamp: protobuf.ToTimestamp(time.Now()), + ID: id, + Data: data, + }}, nil +} + +// UpdateSandbox is not implemented: the sandbox has no resources of its own. +func (s *Service) UpdateSandbox(ctx context.Context, r *api.UpdateSandboxRequest) (*api.UpdateSandboxResponse, error) { + return nil, errgrpc.ToGRPCf(errdefs.ErrNotImplemented, "%s does not implement UpdateSandbox", BinaryName) +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service_test.go b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service_test.go new file mode 100644 index 0000000000..855c36ddd2 --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/sandbox/service_test.go @@ -0,0 +1,142 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + api "github.com/containerd/containerd/api/runtime/sandbox/v1" + "github.com/containerd/errdefs" + "github.com/containerd/errdefs/pkg/errgrpc" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/containerd/containerd/v2/pkg/shutdown" +) + +func TestServiceWithoutSandbox(t *testing.T) { + ctx := context.Background() + _, sd := shutdown.WithShutdown(ctx) + s := NewService(sd) + + // Every sandbox scoped call is NotFound until a sandbox is created. + _, err := s.StartSandbox(ctx, &api.StartSandboxRequest{SandboxID: testID}) + assert.True(t, errdefs.IsNotFound(errgrpc.ToNative(err)), "start: %v", err) + _, err = s.StopSandbox(ctx, &api.StopSandboxRequest{SandboxID: testID}) + assert.True(t, errdefs.IsNotFound(errgrpc.ToNative(err)), "stop: %v", err) + _, err = s.SandboxStatus(ctx, &api.SandboxStatusRequest{SandboxID: testID}) + assert.True(t, errdefs.IsNotFound(errgrpc.ToNative(err)), "status: %v", err) + _, err = s.PingSandbox(ctx, &api.PingRequest{SandboxID: testID}) + assert.True(t, errdefs.IsNotFound(errgrpc.ToNative(err)), "ping: %v", err) + _, err = s.WaitSandbox(ctx, &api.WaitSandboxRequest{SandboxID: testID}) + assert.True(t, errdefs.IsNotFound(errgrpc.ToNative(err)), "wait: %v", err) + _, err = s.SandboxMetrics(ctx, &api.SandboxMetricsRequest{SandboxID: testID}) + assert.True(t, errdefs.IsNotFound(errgrpc.ToNative(err)), "metrics: %v", err) + + _, err = s.UpdateSandbox(ctx, &api.UpdateSandboxRequest{SandboxID: testID}) + assert.True(t, errdefs.IsNotImplemented(errgrpc.ToNative(err)), "update: %v", err) + + resp, err := s.Platform(ctx, &api.PlatformRequest{SandboxID: testID}) + require.NoError(t, err) + assert.Equal(t, "linux", resp.GetPlatform().GetOS()) + + // Shutting down, even an unknown sandbox, ends the shim. + select { + case <-sd.Done(): + t.Fatal("the shim must not shut down before ShutdownSandbox") + default: + } + _, err = s.ShutdownSandbox(ctx, &api.ShutdownSandboxRequest{SandboxID: testID}) + require.NoError(t, err) + select { + case <-sd.Done(): + case <-time.After(5 * time.Second): + t.Fatal("ShutdownSandbox must end the shim") + } +} + +// TestStopRetriesRelease checks that a sandbox is stopped, and Wait returns, +// even when releasing what it holds fails, and that the next stop retries the +// release. +func TestStopRetriesRelease(t *testing.T) { + ctx := context.Background() + bundle := t.TempDir() + // A file where the pin directory belongs makes Cleanup fail. + pins := filepath.Join(bundle, pinDir) + require.NoError(t, os.WriteFile(pins, nil, 0o600)) + inst := &instance{cfg: &Config{ID: testID}, bundle: bundle, done: make(chan struct{})} + s := &Service{} + + require.Error(t, s.stop(ctx, inst)) + assert.True(t, inst.stopped) + assert.False(t, inst.released) + select { + case <-inst.done: + default: + t.Fatal("a stopped sandbox must end Wait even if its release failed") + } + + require.NoError(t, os.Remove(pins)) + require.NoError(t, s.stop(ctx, inst)) + assert.True(t, inst.released) + require.NoError(t, s.stop(ctx, inst), "stopping a released sandbox is a no-op") +} + +func TestSandboxStatusBeforeStart(t *testing.T) { + ctx := context.Background() + _, sd := shutdown.WithShutdown(ctx) + s := NewService(sd) + s.sandbox = &instance{cfg: &Config{ID: testID}, done: make(chan struct{})} + + // A created sandbox is not ready until it is started. + status, err := s.SandboxStatus(ctx, &api.SandboxStatusRequest{SandboxID: testID}) + require.NoError(t, err) + assert.Equal(t, runtime.PodSandboxState_SANDBOX_NOTREADY.String(), status.GetState()) + assert.Nil(t, status.GetExitedAt()) + + _, err = s.StartSandbox(ctx, &api.StartSandboxRequest{SandboxID: testID}) + require.NoError(t, err) + status, err = s.SandboxStatus(ctx, &api.SandboxStatusRequest{SandboxID: testID}) + require.NoError(t, err) + assert.Equal(t, runtime.PodSandboxState_SANDBOX_READY.String(), status.GetState()) +} + +func TestShutdownSandboxOther(t *testing.T) { + ctx := context.Background() + _, sd := shutdown.WithShutdown(ctx) + s := NewService(sd) + s.sandbox = &instance{cfg: &Config{ID: testID}, done: make(chan struct{})} + + // A shutdown meant for another sandbox must neither stop the hosted one + // nor end the shim. + _, err := s.ShutdownSandbox(ctx, &api.ShutdownSandboxRequest{SandboxID: "other"}) + assert.True(t, errdefs.IsNotFound(errgrpc.ToNative(err)), "shutdown: %v", err) + require.NotNil(t, s.sandbox) + assert.False(t, s.sandbox.stopped) + select { + case <-sd.Done(): + t.Fatal("the shim must not shut down for another sandbox") + default: + } +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/task.go b/cmd/containerd-shim-sandboxed-runc-v2/task.go new file mode 100644 index 0000000000..d404399d12 --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/task.go @@ -0,0 +1,100 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + + taskAPI "github.com/containerd/containerd/api/runtime/task/v3" + "github.com/containerd/plugin" + "github.com/containerd/plugin/registry" + "github.com/containerd/ttrpc" + + "github.com/containerd/containerd/v2/cmd/containerd-shim-runc-v2/task" + ptypes "github.com/containerd/containerd/v2/pkg/protobuf/types" + "github.com/containerd/containerd/v2/pkg/shim" + "github.com/containerd/containerd/v2/pkg/shutdown" + "github.com/containerd/containerd/v2/plugins" +) + +// The containers of the pod run through the task service of +// containerd-shim-runc-v2, the very same implementation: this file only wraps +// it. Nothing about how containers are created, started, exec'd or deleted +// changes. +// +// The one thing the wrapper overrides is who ends the shim. containerd sends +// the task Shutdown RPC after every task deletion, and the shared task +// service exits the shim as soon as no container is left. That is right for a +// shim that exists for its tasks and wrong for a shim whose sandbox outlives +// them: a pod with no containers is still a pod. Here Shutdown is +// acknowledged and ignored; the shim exits when its sandbox is shut down +// (see the sandbox package). +// +// TODO: consolidate with containerd-shim-runc-v2 so that both shims register +// the same task service without a wrapper, e.g. by letting the host of the +// shared service decide the shim lifetime. Wrapping the shutdown.Service the +// shared service receives (a no-op Shutdown, everything else delegated) would +// work as well; the RPC wrapper is kept because it states the intent in one +// place. + +func init() { + registry.Register(&plugin.Registration{ + Type: plugins.TTRPCPlugin, + ID: "task", + Requires: []plugin.Type{ + plugins.EventPlugin, + plugins.InternalPlugin, + }, + InitFn: func(ic *plugin.InitContext) (any, error) { + pp, err := ic.GetByID(plugins.EventPlugin, "publisher") + if err != nil { + return nil, err + } + ss, err := ic.GetByID(plugins.InternalPlugin, "shutdown") + if err != nil { + return nil, err + } + svc, err := task.NewTaskService(ic.Context, pp.(shim.Publisher), ss.(shutdown.Service)) + if err != nil { + return nil, err + } + return &taskService{TTRPCTaskService: svc}, nil + }, + }) +} + +// taskService is the task service of containerd-shim-runc-v2 with the shim +// lifetime taken out of its hands. Every RPC but Shutdown is the embedded one. +type taskService struct { + taskAPI.TTRPCTaskService +} + +var _ shim.TTRPCService = (*taskService)(nil) + +// RegisterTTRPC implements shim.TTRPCService. +func (s *taskService) RegisterTTRPC(server *ttrpc.Server) error { + taskAPI.RegisterTTRPCTaskService(server, s) + return nil +} + +// Shutdown acknowledges the request and keeps the shim running: the sandbox +// decides when the shim exits. +func (s *taskService) Shutdown(context.Context, *taskAPI.ShutdownRequest) (*ptypes.Empty, error) { + return &ptypes.Empty{}, nil +} diff --git a/cmd/containerd-shim-sandboxed-runc-v2/task_test.go b/cmd/containerd-shim-sandboxed-runc-v2/task_test.go new file mode 100644 index 0000000000..a17b71e1df --- /dev/null +++ b/cmd/containerd-shim-sandboxed-runc-v2/task_test.go @@ -0,0 +1,53 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + "testing" + + taskAPI "github.com/containerd/containerd/api/runtime/task/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ptypes "github.com/containerd/containerd/v2/pkg/protobuf/types" +) + +// shuttingDown counts the Shutdown calls that reach the wrapped task service. +type shuttingDown struct { + taskAPI.TTRPCTaskService + calls int +} + +func (s *shuttingDown) Shutdown(context.Context, *taskAPI.ShutdownRequest) (*ptypes.Empty, error) { + s.calls++ + return &ptypes.Empty{}, nil +} + +func TestShutdownKeepsTheShimRunning(t *testing.T) { + inner := &shuttingDown{} + svc := &taskService{TTRPCTaskService: inner} + + // containerd sends Shutdown after every task deletion; the shared task + // service would exit the shim once no container is left. The sandbox + // shim must stay up for its pod. + _, err := svc.Shutdown(context.Background(), &taskAPI.ShutdownRequest{ID: "container"}) + require.NoError(t, err) + assert.Equal(t, 0, inner.calls, "the shared task service must not see the shutdown") +} diff --git a/core/sandbox/controller.go b/core/sandbox/controller.go index 2849883daa..7a4b41d203 100644 --- a/core/sandbox/controller.go +++ b/core/sandbox/controller.go @@ -19,10 +19,12 @@ package sandbox import ( "context" "fmt" + "path/filepath" "time" "github.com/containerd/containerd/api/types" "github.com/containerd/containerd/v2/core/mount" + "github.com/containerd/errdefs" "github.com/containerd/typeurl/v2" imagespec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -36,10 +38,28 @@ type CreateOptions struct { Options typeurl.Any NetNSPath string Annotations map[string]string + // RuntimePath is an absolute path to the shim binary to start for the + // sandbox instead of resolving it from the runtime name, the same way a + // task can be created with a runtime path. + RuntimePath string } type CreateOpt func(*CreateOptions) error +// WithRuntimePath starts the sandbox with the shim binary at the given +// absolute path instead of the one resolved from the runtime name. An empty +// path keeps the runtime name; a relative one is rejected, since the shim +// manager would take it for a runtime name. +func WithRuntimePath(path string) CreateOpt { + return func(co *CreateOptions) error { + if path != "" && !filepath.IsAbs(path) { + return fmt.Errorf("runtime path %q is not absolute: %w", path, errdefs.ErrInvalidArgument) + } + co.RuntimePath = path + return nil + } +} + // WithRootFS is used to create a sandbox with the provided rootfs mount func WithRootFS(m []mount.Mount) CreateOpt { return func(co *CreateOptions) error { diff --git a/core/sandbox/controller_test.go b/core/sandbox/controller_test.go new file mode 100644 index 0000000000..d5137f41e3 --- /dev/null +++ b/core/sandbox/controller_test.go @@ -0,0 +1,42 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "path/filepath" + "testing" + + "github.com/containerd/errdefs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWithRuntimePath(t *testing.T) { + var opts CreateOptions + require.NoError(t, WithRuntimePath("")(&opts)) + assert.Empty(t, opts.RuntimePath, "an empty path keeps the runtime name") + + // An absolute path of this platform (a leading slash is not absolute on + // Windows). + abs := filepath.Join(t.TempDir(), "containerd-shim-example-v1") + require.NoError(t, WithRuntimePath(abs)(&opts)) + assert.Equal(t, abs, opts.RuntimePath) + + err := WithRuntimePath("containerd-shim-example-v1")(&opts) + require.Error(t, err, "a relative path would be taken for a runtime name") + assert.True(t, errdefs.IsInvalidArgument(err)) +} diff --git a/core/sandbox/proxy/controller.go b/core/sandbox/proxy/controller.go index 5f8ca44843..2c8ae55ed3 100644 --- a/core/sandbox/proxy/controller.go +++ b/core/sandbox/proxy/controller.go @@ -24,6 +24,7 @@ import ( "github.com/containerd/containerd/api/types" "github.com/containerd/errdefs" "github.com/containerd/errdefs/pkg/errgrpc" + "github.com/containerd/log" "github.com/containerd/typeurl/v2" imagespec "github.com/opencontainers/image-spec/specs-go/v1" @@ -47,7 +48,17 @@ func NewSandboxController(client api.ControllerClient, name string) sandbox.Cont func (s *remoteSandboxController) Create(ctx context.Context, sandboxInfo sandbox.Sandbox, opts ...sandbox.CreateOpt) error { var options sandbox.CreateOptions for _, opt := range opts { - opt(&options) + if err := opt(&options); err != nil { + return err + } + } + if options.RuntimePath != "" { + // ControllerCreateRequest has no field for it; the remote controller + // starts whatever the runtime name resolves to. + log.G(ctx).WithFields(log.Fields{ + "runtime_path": options.RuntimePath, + "runtime": sandboxInfo.Runtime.Name, + }).Warn("runtime path is not forwarded to a remote sandbox controller, which resolves the runtime name instead") } apiSandbox := sandbox.ToProto(&sandboxInfo) _, err := s.client.Create(ctx, &api.ControllerCreateRequest{ diff --git a/core/sandbox/proxy/controller_test.go b/core/sandbox/proxy/controller_test.go new file mode 100644 index 0000000000..78dbb30e4e --- /dev/null +++ b/core/sandbox/proxy/controller_test.go @@ -0,0 +1,35 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package proxy + +import ( + "context" + "testing" + + "github.com/containerd/errdefs" + "github.com/stretchr/testify/require" + + "github.com/containerd/containerd/v2/core/sandbox" +) + +// A failing create option ends Create before the request is sent: the +// controller has no client here, so getting past the options would panic. +func TestCreateOptionError(t *testing.T) { + c := NewSandboxController(nil, "remote") + err := c.Create(context.Background(), sandbox.Sandbox{ID: "sb"}, sandbox.WithRuntimePath("containerd-shim-example-v1")) + require.ErrorIs(t, err, errdefs.ErrInvalidArgument) +} diff --git a/internal/cri/opts/spec_opts.go b/internal/cri/opts/spec_opts.go index 0d78d54673..adb7ce7c04 100644 --- a/internal/cri/opts/spec_opts.go +++ b/internal/cri/opts/spec_opts.go @@ -344,17 +344,103 @@ func WithNamespacePath(t runtimespec.LinuxNamespaceType, nsPath string) oci.Spec } } -// WithPodNamespaces sets the pod namespaces for the container +// PodNamespacePaths are the paths of the pod namespaces a container joins. +// +// A pod sandbox is either PID-backed (a process such as pause is a member of +// every pod namespace, and the paths are derived from its pid with +// PodNamespacePathsFromPid) or path-backed (the sandbox holds the namespaces +// without a process and reports their paths, typically bind-mounted namespace +// files). An empty path means the sandbox does not hold that namespace. +type PodNamespacePaths struct { + Network string + IPC string + UTS string + // PID is the pod PID namespace, joined only when the pod shares it + // (NamespaceMode_POD or NamespaceMode_TARGET). The container keeps its own + // PID namespace otherwise. It is empty for a host PID pod on a sandbox + // without a process: the containers are then in the host PID namespace. + PID string + // PrivatePID is set by a sandbox that holds no PID namespace and declares + // that each container gets a new one, even when the pod asked to share + // (a sandbox without a process has no PID 1 to hold a shared namespace). + // It is an explicit declaration in the sandbox spec, not a fallback. + PrivatePID bool + User string +} + +// PodNamespacePathsFromPid derives the pod namespace paths from the pid of a +// process that is a member of every pod namespace, e.g. the pause container. +// targetPid is the process whose PID namespace the container joins when the +// pod shares one; it differs from sandboxPid for NamespaceMode_TARGET. +func PodNamespacePathsFromPid(sandboxPid, targetPid uint32) PodNamespacePaths { + return PodNamespacePaths{ + Network: GetNetworkNamespace(sandboxPid), + IPC: GetIPCNamespace(sandboxPid), + UTS: GetUTSNamespace(sandboxPid), + PID: GetPIDNamespace(targetPid), + User: GetUserNamespace(sandboxPid), + } +} + +// WithPodNamespaces sets the pod namespaces for the container from the pid of +// the sandbox process. See WithPodNamespacePaths. func WithPodNamespaces(config *runtime.LinuxContainerSecurityContext, sandboxPid uint32, targetPid uint32, uids, gids []runtimespec.LinuxIDMapping) oci.SpecOpts { + return WithPodNamespacePaths(config, config.GetNamespaceOptions(), PodNamespacePathsFromPid(sandboxPid, targetPid), uids, gids) +} + +// WithPodNamespacePaths makes the container join the pod namespaces at the +// given paths. The network, IPC and UTS namespaces of a container are always +// those of its pod, whatever the container config says (with a pause +// container the container joined the namespaces of pause, which was itself in +// the host namespaces the pod asked for); pod is the namespace options of the +// pod sandbox and decides which of them are host namespaces. The PID and user +// namespaces follow the container config as before; a container sharing the +// pod PID namespace (NamespaceMode_POD) is in the host one when the pod is. +// +// Every namespace the container is expected to share with the pod must have a +// path; a missing path is an error rather than a fallback to a fresh or host +// namespace, so that a broken sandbox cannot silently weaken isolation. The +// entry of a host namespace (NamespaceMode_NODE) is removed from the spec +// when the sandbox provides no path for it. +func WithPodNamespacePaths(config *runtime.LinuxContainerSecurityContext, pod *runtime.NamespaceOption, paths PodNamespacePaths, uids, gids []runtimespec.LinuxIDMapping) oci.SpecOpts { namespaces := config.GetNamespaceOptions() - opts := []oci.SpecOpts{ - oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: runtimespec.NetworkNamespace, Path: GetNetworkNamespace(sandboxPid)}), - oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: runtimespec.IPCNamespace, Path: GetIPCNamespace(sandboxPid)}), - oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: runtimespec.UTSNamespace, Path: GetUTSNamespace(sandboxPid)}), + var opts []oci.SpecOpts + join := func(t runtimespec.LinuxNamespaceType, path string, host bool) error { + switch { + case path != "": + opts = append(opts, oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: t, Path: path})) + case host: + opts = append(opts, WithoutNamespace(t)) + default: + return fmt.Errorf("pod sandbox provides no %s namespace for the container to join", t) + } + return nil } - if namespaces.GetPid() != runtime.NamespaceMode_CONTAINER { - opts = append(opts, oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: runtimespec.PIDNamespace, Path: GetPIDNamespace(targetPid)})) + + hostNetwork := pod.GetNetwork() == runtime.NamespaceMode_NODE + if err := join(runtimespec.NetworkNamespace, paths.Network, hostNetwork); err != nil { + return failSpecOpt(err) + } + if err := join(runtimespec.IPCNamespace, paths.IPC, pod.GetIpc() == runtime.NamespaceMode_NODE); err != nil { + return failSpecOpt(err) + } + // There is no CRI option for the UTS namespace: a pod shares it unless it + // uses the host network, in which case it uses the host UTS namespace too. + if err := join(runtimespec.UTSNamespace, paths.UTS, hostNetwork); err != nil { + return failSpecOpt(err) + } + // The default spec already gives the container its own PID namespace, + // which is what NamespaceMode_CONTAINER asks for and what a sandbox that + // declared it holds no PID namespace to share provides. A container that + // shares the pod PID namespace is in the host one when the pod is (pause + // was): a sandbox without a process provides no path for it. + pidMode := namespaces.GetPid() + if pidMode != runtime.NamespaceMode_CONTAINER && (pidMode != runtime.NamespaceMode_POD || !paths.PrivatePID) { + hostPID := pidMode == runtime.NamespaceMode_NODE || (pidMode == runtime.NamespaceMode_POD && pod.GetPid() == runtime.NamespaceMode_NODE) + if err := join(runtimespec.PIDNamespace, paths.PID, hostPID); err != nil { + return failSpecOpt(err) + } } if namespaces.GetUsernsOptions() != nil { @@ -362,7 +448,9 @@ func WithPodNamespaces(config *runtime.LinuxContainerSecurityContext, sandboxPid case runtime.NamespaceMode_NODE: // Nothing to do. Not adding userns field uses the node userns. case runtime.NamespaceMode_POD: - opts = append(opts, oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: runtimespec.UserNamespace, Path: GetUserNamespace(sandboxPid)})) + if err := join(runtimespec.UserNamespace, paths.User, false); err != nil { + return failSpecOpt(err) + } opts = append(opts, oci.WithUserNamespace(uids, gids)) } } @@ -370,6 +458,12 @@ func WithPodNamespaces(config *runtime.LinuxContainerSecurityContext, sandboxPid return oci.Compose(opts...) } +func failSpecOpt(err error) oci.SpecOpts { + return func(context.Context, oci.Client, *containers.Container, *runtimespec.Spec) error { + return err + } +} + const ( // netNSFormat is the format of network namespace of a process. netNSFormat = "/proc/%v/ns/net" diff --git a/internal/cri/opts/spec_opts_namespaces_test.go b/internal/cri/opts/spec_opts_namespaces_test.go new file mode 100644 index 0000000000..6d9c568269 --- /dev/null +++ b/internal/cri/opts/spec_opts_namespaces_test.go @@ -0,0 +1,247 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package opts + +import ( + "context" + "testing" + + runtimespec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/containerd/containerd/v2/pkg/oci" +) + +// defaultNamespaces are the namespaces of the default OCI spec: private, no +// path, before the pod namespaces are applied. +func defaultNamespaces() *runtimespec.Spec { + return &runtimespec.Spec{Linux: &runtimespec.Linux{Namespaces: []runtimespec.LinuxNamespace{ + {Type: runtimespec.PIDNamespace}, + {Type: runtimespec.IPCNamespace}, + {Type: runtimespec.UTSNamespace}, + {Type: runtimespec.MountNamespace}, + {Type: runtimespec.NetworkNamespace}, + }}} +} + +func applyNamespaces(t *testing.T, opt oci.SpecOpts) (*runtimespec.Spec, error) { + t.Helper() + spec := defaultNamespaces() + err := opt(context.Background(), nil, nil, spec) + return spec, err +} + +func namespacePaths(spec *runtimespec.Spec) map[runtimespec.LinuxNamespaceType]string { + paths := make(map[runtimespec.LinuxNamespaceType]string) + for _, ns := range spec.Linux.Namespaces { + paths[ns.Type] = ns.Path + } + return paths +} + +func securityContext(network, ipc, pid runtime.NamespaceMode) *runtime.LinuxContainerSecurityContext { + return &runtime.LinuxContainerSecurityContext{NamespaceOptions: &runtime.NamespaceOption{ + Network: network, + Ipc: ipc, + Pid: pid, + }} +} + +func TestWithPodNamespacePathsFromPid(t *testing.T) { + // A sandbox with a process (pause) keeps today's construction exactly, + // whatever the requested modes: the container joins the namespaces of the + // sandbox process, which is itself in the host namespaces the pod asked for. + for name, sc := range map[string]*runtime.LinuxContainerSecurityContext{ + "pod namespaces": securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_POD), + "host namespaces": securityContext(runtime.NamespaceMode_NODE, runtime.NamespaceMode_NODE, runtime.NamespaceMode_NODE), + "container pid": securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER), + } { + t.Run(name, func(t *testing.T) { + legacy, err := applyNamespaces(t, WithPodNamespaces(sc, 42, 42, nil, nil)) + require.NoError(t, err) + byPath, err := applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), PodNamespacePathsFromPid(42, 42), nil, nil)) + require.NoError(t, err) + assert.Equal(t, legacy, byPath) + + paths := namespacePaths(byPath) + assert.Equal(t, "/proc/42/ns/net", paths[runtimespec.NetworkNamespace]) + assert.Equal(t, "/proc/42/ns/ipc", paths[runtimespec.IPCNamespace]) + assert.Equal(t, "/proc/42/ns/uts", paths[runtimespec.UTSNamespace]) + if sc.GetNamespaceOptions().GetPid() == runtime.NamespaceMode_CONTAINER { + assert.Equal(t, "", paths[runtimespec.PIDNamespace], "the container keeps its own PID namespace") + } else { + assert.Equal(t, "/proc/42/ns/pid", paths[runtimespec.PIDNamespace]) + } + }) + } + + t.Run("target pid", func(t *testing.T) { + sc := securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_TARGET) + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), PodNamespacePathsFromPid(42, 7), nil, nil)) + require.NoError(t, err) + assert.Equal(t, "/proc/7/ns/pid", namespacePaths(spec)[runtimespec.PIDNamespace]) + }) +} + +func TestWithPodNamespacePaths(t *testing.T) { + pathBacked := PodNamespacePaths{ + Network: "/var/run/netns/cni-1", + IPC: "/run/sandbox/ns/ipc", + UTS: "/run/sandbox/ns/uts", + } + + t.Run("pod namespaces by path", func(t *testing.T) { + sc := securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER) + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), pathBacked, nil, nil)) + require.NoError(t, err) + assert.Equal(t, map[runtimespec.LinuxNamespaceType]string{ + runtimespec.NetworkNamespace: pathBacked.Network, + runtimespec.IPCNamespace: pathBacked.IPC, + runtimespec.UTSNamespace: pathBacked.UTS, + runtimespec.PIDNamespace: "", + runtimespec.MountNamespace: "", + }, namespacePaths(spec)) + }) + + t.Run("host namespaces without paths are removed from the spec", func(t *testing.T) { + sc := securityContext(runtime.NamespaceMode_NODE, runtime.NamespaceMode_NODE, runtime.NamespaceMode_NODE) + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), PodNamespacePaths{}, nil, nil)) + require.NoError(t, err) + assert.Equal(t, map[runtimespec.LinuxNamespaceType]string{ + runtimespec.MountNamespace: "", + }, namespacePaths(spec), "only the private mount namespace is left") + }) + + t.Run("a sandbox declaring private PID namespaces", func(t *testing.T) { + // A pod asking to share its PID namespace on a sandbox that holds + // none: the sandbox said so explicitly, the container keeps its own. + sc := securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_POD) + paths := pathBacked + paths.PrivatePID = true + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), paths, nil, nil)) + require.NoError(t, err) + assert.Equal(t, "", namespacePaths(spec)[runtimespec.PIDNamespace]) + assert.Contains(t, namespacePaths(spec), runtimespec.PIDNamespace) + + // The declaration does not cover the host PID namespace. + sc = securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_NODE) + spec, err = applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), paths, nil, nil)) + require.NoError(t, err) + assert.NotContains(t, namespacePaths(spec), runtimespec.PIDNamespace) + + // Nor a container joining the PID namespace of another container. + sc = securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_TARGET) + paths.PID = "/proc/7/ns/pid" + spec, err = applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), paths, nil, nil)) + require.NoError(t, err) + assert.Equal(t, paths.PID, namespacePaths(spec)[runtimespec.PIDNamespace]) + }) + + t.Run("host pid pod", func(t *testing.T) { + // A sandbox without a process holds no PID namespace for a host PID + // pod and provides no path: a container sharing the pod PID namespace + // (NamespaceMode_POD, the zero value) is in the host one like the + // pod, as it was with pause; NamespaceMode_CONTAINER keeps its own. + pod := &runtime.NamespaceOption{Network: runtime.NamespaceMode_POD, Ipc: runtime.NamespaceMode_POD, Pid: runtime.NamespaceMode_NODE} + for _, mode := range []runtime.NamespaceMode{runtime.NamespaceMode_POD, runtime.NamespaceMode_NODE} { + sc := securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, mode) + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, pod, pathBacked, nil, nil)) + require.NoError(t, err, "pid mode %s", mode) + assert.NotContains(t, namespacePaths(spec), runtimespec.PIDNamespace, "pid mode %s", mode) + } + sc := securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER) + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, pod, pathBacked, nil, nil)) + require.NoError(t, err) + assert.Equal(t, "", namespacePaths(spec)[runtimespec.PIDNamespace]) + assert.Contains(t, namespacePaths(spec), runtimespec.PIDNamespace) + }) + + t.Run("host network with pod ipc", func(t *testing.T) { + sc := securityContext(runtime.NamespaceMode_NODE, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER) + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), PodNamespacePaths{IPC: pathBacked.IPC}, nil, nil)) + require.NoError(t, err) + assert.Equal(t, map[runtimespec.LinuxNamespaceType]string{ + runtimespec.IPCNamespace: pathBacked.IPC, + runtimespec.PIDNamespace: "", + runtimespec.MountNamespace: "", + }, namespacePaths(spec), "no network and no UTS namespace entry") + }) + + t.Run("the pod decides the host namespaces, not the container config", func(t *testing.T) { + // A container config that says nothing (NamespaceMode_POD, the zero + // value) in a host network pod: the container is in the host network + // and UTS namespaces like the pod, as it was with pause. + container := securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER) + pod := &runtime.NamespaceOption{Network: runtime.NamespaceMode_NODE, Ipc: runtime.NamespaceMode_POD} + spec, err := applyNamespaces(t, WithPodNamespacePaths(container, pod, PodNamespacePaths{IPC: pathBacked.IPC}, nil, nil)) + require.NoError(t, err) + assert.Equal(t, map[runtimespec.LinuxNamespaceType]string{ + runtimespec.IPCNamespace: pathBacked.IPC, + runtimespec.PIDNamespace: "", + runtimespec.MountNamespace: "", + }, namespacePaths(spec)) + }) + + // A pod namespace the sandbox does not provide is an error, never a + // silent fallback to a fresh or host namespace. + for name, tc := range map[string]struct { + sc *runtime.LinuxContainerSecurityContext + paths PodNamespacePaths + }{ + "missing network": {securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER), PodNamespacePaths{IPC: pathBacked.IPC, UTS: pathBacked.UTS}}, + "missing ipc": {securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER), PodNamespacePaths{Network: pathBacked.Network, UTS: pathBacked.UTS}}, + "missing uts": {securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_CONTAINER), PodNamespacePaths{Network: pathBacked.Network, IPC: pathBacked.IPC}}, + "missing pid": {securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_POD), pathBacked}, + "missing target pid": {securityContext(runtime.NamespaceMode_POD, runtime.NamespaceMode_POD, runtime.NamespaceMode_TARGET), PodNamespacePaths{ + Network: pathBacked.Network, + IPC: pathBacked.IPC, + UTS: pathBacked.UTS, + PrivatePID: true, + }}, + "missing user": {&runtime.LinuxContainerSecurityContext{NamespaceOptions: &runtime.NamespaceOption{ + Network: runtime.NamespaceMode_POD, + Ipc: runtime.NamespaceMode_POD, + Pid: runtime.NamespaceMode_CONTAINER, + UsernsOptions: &runtime.UserNamespace{Mode: runtime.NamespaceMode_POD}, + }}, pathBacked}, + } { + t.Run(name, func(t *testing.T) { + _, err := applyNamespaces(t, WithPodNamespacePaths(tc.sc, tc.sc.GetNamespaceOptions(), tc.paths, nil, nil)) + require.Error(t, err) + assert.Contains(t, err.Error(), "provides no") + }) + } + + t.Run("pod user namespace by path", func(t *testing.T) { + sc := &runtime.LinuxContainerSecurityContext{NamespaceOptions: &runtime.NamespaceOption{ + Network: runtime.NamespaceMode_POD, + Ipc: runtime.NamespaceMode_POD, + Pid: runtime.NamespaceMode_CONTAINER, + UsernsOptions: &runtime.UserNamespace{Mode: runtime.NamespaceMode_POD}, + }} + paths := pathBacked + paths.User = "/run/sandbox/ns/user" + uids := []runtimespec.LinuxIDMapping{{ContainerID: 0, HostID: 65536, Size: 65536}} + spec, err := applyNamespaces(t, WithPodNamespacePaths(sc, sc.GetNamespaceOptions(), paths, uids, uids)) + require.NoError(t, err) + assert.Equal(t, paths.User, namespacePaths(spec)[runtimespec.UserNamespace]) + assert.Equal(t, uids, spec.Linux.UIDMappings) + assert.Equal(t, uids, spec.Linux.GIDMappings) + }) +} diff --git a/internal/cri/sandboxfiles/sandboxfiles.go b/internal/cri/sandboxfiles/sandboxfiles.go new file mode 100644 index 0000000000..3b8f38cb3a --- /dev/null +++ b/internal/cri/sandboxfiles/sandboxfiles.go @@ -0,0 +1,88 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package sandboxfiles builds the content of the files a pod sandbox shares +// with every container of the pod: /etc/hostname, /etc/hosts, /etc/resolv.conf +// and the /dev/shm tmpfs. +// +// The package is deliberately a leaf: it has no dependency on the CRI server +// or on any sandbox implementation, so that both the in-process podsandbox +// controller and a shim that implements the Sandbox API can produce the same +// files. Where the files live and how they reach the containers is decided by +// the caller. +package sandboxfiles + +import ( + "fmt" + "strings" +) + +const ( + // HostnameFile is the file name of the pod hostname file. + HostnameFile = "hostname" + // HostsFile is the file name of the pod hosts file. + HostsFile = "hosts" + // ResolvConfFile is the file name of the pod resolv.conf. + ResolvConfFile = "resolv.conf" + // ShmDir is the directory name of the pod /dev/shm tmpfs. + ShmDir = "shm" + + // EtcHostname is the path of the hostname file inside a container. + EtcHostname = "/etc/hostname" + // EtcHosts is the path of the hosts file on the host and inside a container. + EtcHosts = "/etc/hosts" + // ResolvConfPath is the path of resolv.conf on the host and inside a container. + ResolvConfPath = "/etc/resolv.conf" + // DevShm is the path of the shared memory tmpfs on the host and inside a container. + DevShm = "/dev/shm" + + // DefaultShmSize is the default size of the pod /dev/shm tmpfs. + DefaultShmSize = int64(1024 * 1024 * 64) +) + +// HostnameContent returns the content of the pod hostname file. +func HostnameContent(hostname string) []byte { + return []byte(hostname + "\n") +} + +// ResolvConfContent renders resolv.conf content from CRI DNS options. When no +// option is specified the result is empty. +func ResolvConfContent(servers, searches, options []string) []byte { + var b strings.Builder + + if len(searches) > 0 { + fmt.Fprintf(&b, "search %s\n", strings.Join(searches, " ")) + } + + if len(servers) > 0 { + fmt.Fprintf(&b, "nameserver %s\n", strings.Join(servers, "\nnameserver ")) + } + + if len(options) > 0 { + fmt.Fprintf(&b, "options %s\n", strings.Join(options, " ")) + } + + return []byte(b.String()) +} + +// ShmMountData returns the tmpfs mount data for a pod /dev/shm of the given +// size. A non-positive size falls back to DefaultShmSize. +func ShmMountData(size int64) string { + if size <= 0 { + size = DefaultShmSize + } + return fmt.Sprintf("mode=1777,size=%d", size) +} diff --git a/internal/cri/sandboxfiles/sandboxfiles_test.go b/internal/cri/sandboxfiles/sandboxfiles_test.go new file mode 100644 index 0000000000..e083d41e12 --- /dev/null +++ b/internal/cri/sandboxfiles/sandboxfiles_test.go @@ -0,0 +1,64 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandboxfiles + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestResolvConfContent(t *testing.T) { + for _, test := range []struct { + desc string + servers []string + searches []string + options []string + expected string + }{ + { + desc: "empty dns options should return empty content", + expected: "", + }, + { + desc: "non-empty dns options should return correct content", + servers: []string{"8.8.8.8", "8.8.4.4"}, + searches: []string{"114.114.114.114"}, + options: []string{"timeout:1"}, + expected: "search 114.114.114.114\nnameserver 8.8.8.8\nnameserver 8.8.4.4\noptions timeout:1\n", + }, + { + desc: "servers only", + servers: []string{"1.1.1.1"}, + expected: "nameserver 1.1.1.1\n", + }, + } { + t.Run(test.desc, func(t *testing.T) { + assert.Equal(t, test.expected, string(ResolvConfContent(test.servers, test.searches, test.options))) + }) + } +} + +func TestHostnameContent(t *testing.T) { + assert.Equal(t, "pod-1\n", string(HostnameContent("pod-1"))) +} + +func TestShmMountData(t *testing.T) { + assert.Equal(t, "mode=1777,size=67108864", ShmMountData(0)) + assert.Equal(t, "mode=1777,size=67108864", ShmMountData(-1)) + assert.Equal(t, "mode=1777,size=1024", ShmMountData(1024)) +} diff --git a/internal/cri/server/container_create.go b/internal/cri/server/container_create.go index a3864be214..17cc9deac4 100644 --- a/internal/cri/server/container_create.go +++ b/internal/cri/server/container_create.go @@ -78,6 +78,15 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta if sandbox.Status.Get().State != sandboxstore.StateReady { return nil, fmt.Errorf("sandbox container %q is not running", sandboxID) } + // A sandbox without a process (pid 0) describes the namespaces and files it + // holds in the spec it returned on start; containers are wired up from it. + var sandboxSpec *runtimespec.Spec + if sandboxPid == 0 { + sandboxSpec, err = c.sandboxSpec(ctx, sandboxID) + if err != nil { + return nil, err + } + } span.SetAttributes( tracing.Attribute("sandbox.id", sandboxID), tracing.Attribute("sandbox.pid", sandboxPid), @@ -153,6 +162,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta podSandboxConfig: sandboxConfig, sandboxRuntimeHandler: sandbox.Metadata.RuntimeHandler, sandboxPid: sandboxPid, + sandboxSpec: sandboxSpec, NetNSPath: sandbox.NetNSPath, containerName: containerName, containerdImage: &containerdImage, @@ -183,6 +193,10 @@ type createContainerRequest struct { containerdImage *containerd.Image meta *containerstore.Metadata start time.Time + // sandboxSpec is the spec returned by the sandbox controller on start. It is + // only loaded for sandboxes without a process (sandboxPid == 0), where it is + // the source of the pod namespace paths and shared file mounts. + sandboxSpec *runtimespec.Spec } func (c *criService) createContainer(r *createContainerRequest) (_ string, retErr error) { @@ -275,6 +289,7 @@ func (c *criService) createContainer(r *createContainerRequest) (_ string, retEr volumeMounts, ociRuntime, runtimeHandler, + r.sandboxSpec, ) if err != nil { return "", fmt.Errorf("failed to generate container %q spec: %w", r.containerID, err) @@ -612,6 +627,7 @@ func (c *criService) buildContainerSpec( extraMounts []*runtime.Mount, ociRuntime criconfig.Runtime, runtimeHandler *runtime.RuntimeHandler, + sandboxSpec *runtimespec.Spec, ) (_ *runtimespec.Spec, retErr error) { var ( specOpts []oci.SpecOpts @@ -627,12 +643,13 @@ func (c *criService) buildContainerSpec( case isLinux: // Generate container mounts. // No mounts are passed for other platforms. - linuxMounts := c.linuxContainerMounts(sandboxID, config) + linuxMounts := c.linuxContainerMounts(sandboxID, config, sandboxSpec) specOpts, err = c.buildLinuxSpec( id, sandboxID, sandboxPid, + sandboxSpec, containerName, imageName, config, @@ -680,6 +697,7 @@ func (c *criService) buildLinuxSpec( id string, sandboxID string, sandboxPid uint32, + sandboxSpec *runtimespec.Spec, containerName string, imageName string, config *runtime.ContainerConfig, @@ -922,7 +940,7 @@ func (c *criService) buildLinuxSpec( specOpts = append(specOpts, customopts.WithOOMScoreAdj(config, c.config.RestrictOOMScoreAdj), - customopts.WithPodNamespaces(securityContext, sandboxPid, targetPid, uids, gids), + customopts.WithPodNamespacePaths(securityContext, sandboxConfig.GetLinux().GetSecurityContext().GetNamespaceOptions(), podNamespacePaths(sandboxPid, targetPid, sandboxSpec), uids, gids), customopts.WithSupplementalGroups(supplementalGroups), ) specOpts = append( @@ -1084,9 +1102,94 @@ func (c *criService) buildDarwinSpec( return specOpts, nil } +// sandboxSpec loads the OCI spec the sandbox controller returned when the +// sandbox was started. It is nil when the controller returned none. +func (c *criService) sandboxSpec(ctx context.Context, sandboxID string) (*runtimespec.Spec, error) { + sandboxInfo, err := c.client.SandboxStore().Get(ctx, sandboxID) + if err != nil { + return nil, fmt.Errorf("failed to get sandbox %q from metadata store: %w", sandboxID, err) + } + if sandboxInfo.Spec == nil { + return nil, nil + } + var spec runtimespec.Spec + if err := typeurl.UnmarshalTo(sandboxInfo.Spec, &spec); err != nil { + return nil, fmt.Errorf("failed to unmarshal spec of sandbox %q: %w", sandboxID, err) + } + return &spec, nil +} + +// podNamespacePaths resolves the namespaces a container joins. +// +// A PID-backed sandbox (pause) is a member of every pod namespace, so the +// paths are derived from its pid exactly as before. A sandbox without a +// process holds the pod namespaces itself and lists their paths in the Linux +// section of the spec it returned on start; a namespace missing from that +// spec is one the sandbox does not hold, which WithPodNamespacePaths turns into +// the host namespace when the pod asked for it and into an error otherwise. +// +// A sandbox that reports no process and returns no spec (or none with a Linux +// section) is a pre-existing Sandbox API shim that describes neither (VM based +// shims ignore the paths CRI generates); those keep the historical pid derived +// construction so that their behavior does not change. +func podNamespacePaths(sandboxPid, targetPid uint32, sandboxSpec *runtimespec.Spec) customopts.PodNamespacePaths { + if sandboxPid != 0 || sandboxSpec == nil || sandboxSpec.Linux == nil { + return customopts.PodNamespacePathsFromPid(sandboxPid, targetPid) + } + var paths customopts.PodNamespacePaths + for _, ns := range sandboxSpec.Linux.Namespaces { + switch ns.Type { + case runtimespec.NetworkNamespace: + paths.Network = ns.Path + case runtimespec.IPCNamespace: + paths.IPC = ns.Path + case runtimespec.UTSNamespace: + paths.UTS = ns.Path + case runtimespec.PIDNamespace: + // A PID namespace entry without a path is the sandbox declaring + // that it holds none and that each container gets a new one. + paths.PID = ns.Path + paths.PrivatePID = ns.Path == "" + case runtimespec.UserNamespace: + paths.User = ns.Path + } + } + // A container targeting another container's PID namespace joins that + // container, not the sandbox. + if targetPid != sandboxPid { + paths.PID = customopts.GetPIDNamespace(targetPid) + } + return paths +} + +// sandboxSharedFile returns the host path of a pod shared file, i.e. a file +// the sandbox provides to every container of the pod at destination (for +// example /etc/hosts). The fixed location under the CRI sandbox directory +// wins when it exists; otherwise the mount the sandbox declared for that +// destination in its spec is used, which is how a sandbox implementation that +// owns the files, such as a Sandbox API shim, hands them over. The result is +// empty when neither exists. +func (c *criService) sandboxSharedFile(fixedPath, destination string, sandboxSpec *runtimespec.Spec) string { + if _, err := c.os.Stat(fixedPath); err == nil { + return fixedPath + } + if sandboxSpec == nil { + return "" + } + for _, m := range sandboxSpec.Mounts { + if m.Destination != destination || m.Source == "" { + continue + } + if _, err := c.os.Stat(m.Source); err == nil { + return m.Source + } + } + return "" +} + // linuxContainerMounts sets up necessary container system file mounts // including /dev/shm, /etc/hosts and /etc/resolv.conf. -func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.ContainerConfig) []*runtime.Mount { +func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.ContainerConfig, sandboxSpec *runtimespec.Spec) []*runtime.Mount { var mounts []*runtime.Mount securityContext := config.GetLinux().GetSecurityContext() var uidMappings, gidMappings []*runtime.IDMapping @@ -1101,8 +1204,7 @@ func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.Cont // do not mount this in that case. // TODO(random-liu): Remove the check and always mount this when // containerd 1.1 and 1.2 are deprecated. - hostpath := c.getSandboxHostname(sandboxID) - if _, err := c.os.Stat(hostpath); err == nil { + if hostpath := c.sandboxSharedFile(c.getSandboxHostname(sandboxID), etcHostname, sandboxSpec); hostpath != "" { mounts = append(mounts, &runtime.Mount{ ContainerPath: etcHostname, HostPath: hostpath, @@ -1115,10 +1217,9 @@ func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.Cont } if !isInCRIMounts(etcHosts, config.GetMounts()) { - hostpath := c.getSandboxHosts(sandboxID) // /etc/hosts could be delegated to remote sandbox controller. That file isn't required to be existed // in host side for some sandbox runtimes. Skip it if we don't need it. - if _, err := c.os.Stat(hostpath); err == nil { + if hostpath := c.sandboxSharedFile(c.getSandboxHosts(sandboxID), etcHosts, sandboxSpec); hostpath != "" { mounts = append(mounts, &runtime.Mount{ ContainerPath: etcHosts, HostPath: hostpath, @@ -1133,10 +1234,9 @@ func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.Cont // Mount sandbox resolv.config. // TODO: Need to figure out whether we should always mount it as read-only if !isInCRIMounts(resolvConfPath, config.GetMounts()) { - hostpath := c.getResolvPath(sandboxID) // The ownership of /etc/resolv.conf could be delegated to remote sandbox controller. That file isn't // required to be existed in host side for some sandbox runtimes. Skip it if we don't need it. - if _, err := c.os.Stat(hostpath); err == nil { + if hostpath := c.sandboxSharedFile(c.getResolvPath(sandboxID), resolvConfPath, sandboxSpec); hostpath != "" { mounts = append(mounts, &runtime.Mount{ ContainerPath: resolvConfPath, HostPath: hostpath, @@ -1152,6 +1252,8 @@ func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.Cont sandboxDevShm := c.getSandboxDevShm(sandboxID) if securityContext.GetNamespaceOptions().GetIpc() == runtime.NamespaceMode_NODE { sandboxDevShm = devShm + } else if source := c.sandboxSharedFile(sandboxDevShm, devShm, sandboxSpec); source != "" { + sandboxDevShm = source } // The ownership of /dev/shm could be delegated to remote sandbox controller. That file isn't required // to be existed in host side for some sandbox runtimes. Skip it if we don't need it. diff --git a/internal/cri/server/container_create_linux_test.go b/internal/cri/server/container_create_linux_test.go index aa23fd81ac..271a215e72 100644 --- a/internal/cri/server/container_create_linux_test.go +++ b/internal/cri/server/container_create_linux_test.go @@ -251,7 +251,7 @@ func TestContainerCapabilities(t *testing.T) { c.allCaps = allCaps containerConfig.Linux.SecurityContext.Capabilities = test.capability - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) if selinux.GetEnabled() { @@ -286,7 +286,7 @@ func TestContainerSpecTty(t *testing.T) { c := newTestCRIService() for _, tty := range []bool{true, false} { containerConfig.Tty = tty - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) assert.Equal(t, tty, spec.Process.Terminal) @@ -313,7 +313,7 @@ func TestContainerSpecDefaultPath(t *testing.T) { imageConfig.Env = append(imageConfig.Env, pathenv) expected = pathenv } - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) assert.Contains(t, spec.Process.Env, expected) @@ -330,7 +330,7 @@ func TestContainerSpecReadonlyRootfs(t *testing.T) { c := newTestCRIService() for _, readonly := range []bool{true, false} { containerConfig.Linux.SecurityContext.ReadonlyRootfs = readonly - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) assert.Equal(t, readonly, spec.Root.Readonly) @@ -364,7 +364,7 @@ func TestContainerSpecWithExtraMounts(t *testing.T) { Readonly: false, }, } - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, extraMounts, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, extraMounts, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) var mounts, sysMounts []runtimespec.Mount @@ -430,7 +430,7 @@ func TestContainerAndSandboxPrivileged(t *testing.T) { sandboxConfig.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ Privileged: test.sandboxPrivileged, } - _, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + _, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) if test.expectError { assert.Error(t, err) } else { @@ -470,7 +470,7 @@ func TestPrivilegedBindMount(t *testing.T) { containerConfig.Linux.SecurityContext.Privileged = test.privileged sandboxConfig.Linux.SecurityContext.Privileged = test.privileged - spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) if test.expectedSysFSRO { @@ -542,7 +542,7 @@ func TestCgroupNamespace(t *testing.T) { containerConfig.Linux.SecurityContext.Privileged = tt.privileged sandboxConfig.Linux.SecurityContext.Privileged = tt.privileged - spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) hasCgroupNS := false @@ -711,7 +711,7 @@ func TestPidNamespace(t *testing.T) { } { t.Run(test.desc, func(t *testing.T) { containerConfig.Linux.SecurityContext.NamespaceOptions = &runtime.NamespaceOption{Pid: test.pidNS} - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) assert.Contains(t, spec.Linux.Namespaces, test.expected) }) @@ -885,7 +885,7 @@ func TestUserNamespace(t *testing.T) { sandboxUserns = test.sandboxUserNS } sandboxConfig.Linux.SecurityContext.NamespaceOptions = &runtime.NamespaceOption{UsernsOptions: sandboxUserns} - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) if test.err { require.Error(t, err) @@ -915,7 +915,7 @@ func TestNoDefaultRunMount(t *testing.T) { ociRuntime := config.Runtime{} c := newTestCRIService() - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) for _, mount := range spec.Mounts { assert.NotEqual(t, "/run", mount.Destination) @@ -1008,7 +1008,7 @@ func TestMaskedAndReadonlyPaths(t *testing.T) { sandboxConfig.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ Privileged: test.privileged, } - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) if !test.privileged { // specCheck presumes an unprivileged container specCheck(t, testID, testSandboxID, testPid, spec) @@ -1060,7 +1060,7 @@ func TestHostname(t *testing.T) { sandboxConfig.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ NamespaceOptions: &runtime.NamespaceOption{Network: test.networkNs}, } - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) assert.Contains(t, spec.Process.Env, test.expectedEnv) @@ -1169,7 +1169,7 @@ additional-group-for-root:x:22222:root containerConfig.Linux.SecurityContext = test.securityContext imageConfig.User = test.imageConfigUser - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) spec.Root.Path = tempRootDir // simulating /etc/{passwd, group} @@ -1244,7 +1244,7 @@ func TestNonRootUserAndDevices(t *testing.T) { }, } - spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil) + spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil, nil) assert.NoError(t, err) assert.Equal(t, test.expectedDeviceUID, *spec.Linux.Devices[0].UID) @@ -1317,7 +1317,7 @@ func TestPrivilegedDevices(t *testing.T) { PrivilegedWithoutHostDevices: test.privilegedWithoutHostDevices, PrivilegedWithoutHostDevicesAllDevicesAllowed: test.privilegedWithoutHostDevicesAllDevicesAllowed, } - spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) hostDevicesRaw, err := oci.HostDevices() @@ -1372,7 +1372,7 @@ func TestBaseOCISpec(t *testing.T) { testPid := uint32(1234) containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData() - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) @@ -1700,7 +1700,7 @@ containerEdits: }, } { t.Run(test.description, func(t *testing.T) { - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) @@ -1816,7 +1816,7 @@ func TestUserNamespaceWithHostNetwork(t *testing.T) { }, } - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) require.NotNil(t, spec) diff --git a/internal/cri/server/container_create_sandboxspec_linux_test.go b/internal/cri/server/container_create_sandboxspec_linux_test.go new file mode 100644 index 0000000000..d71a5d17ef --- /dev/null +++ b/internal/cri/server/container_create_sandboxspec_linux_test.go @@ -0,0 +1,168 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package server + +import ( + "os" + "testing" + + runtimespec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + + customopts "github.com/containerd/containerd/v2/internal/cri/opts" + ostesting "github.com/containerd/containerd/v2/pkg/os/testing" +) + +// pathBackedSandboxSpec is what a sandbox without a process returns on start: +// the namespaces it holds and the pod shared files it owns. +func pathBackedSandboxSpec() *runtimespec.Spec { + return &runtimespec.Spec{ + Linux: &runtimespec.Linux{ + Namespaces: []runtimespec.LinuxNamespace{ + {Type: runtimespec.NetworkNamespace, Path: "/var/run/netns/cni-1"}, + {Type: runtimespec.IPCNamespace, Path: "/run/sandbox/ns/ipc"}, + {Type: runtimespec.UTSNamespace, Path: "/run/sandbox/ns/uts"}, + }, + }, + Mounts: []runtimespec.Mount{ + {Destination: "/etc/hostname", Type: "bind", Source: "/run/sandbox/hostname"}, + {Destination: "/etc/hosts", Type: "bind", Source: "/run/sandbox/hosts"}, + {Destination: "/etc/resolv.conf", Type: "bind", Source: "/run/sandbox/resolv.conf"}, + {Destination: "/dev/shm", Type: "bind", Source: "/run/sandbox/shm"}, + }, + } +} + +func TestPodNamespacePaths(t *testing.T) { + spec := pathBackedSandboxSpec() + for name, tc := range map[string]struct { + sandboxPid, targetPid uint32 + spec *runtimespec.Spec + expected customopts.PodNamespacePaths + }{ + "a sandbox process wins over the spec": { + sandboxPid: 42, targetPid: 42, spec: spec, + expected: customopts.PodNamespacePathsFromPid(42, 42), + }, + "no process and no spec is a legacy sandbox api shim": { + expected: customopts.PodNamespacePathsFromPid(0, 0), + }, + "no process and a spec without a linux section is legacy too": { + spec: &runtimespec.Spec{}, + expected: customopts.PodNamespacePathsFromPid(0, 0), + }, + "no process and a spec with namespaces is path backed": { + spec: spec, + expected: customopts.PodNamespacePaths{ + Network: "/var/run/netns/cni-1", + IPC: "/run/sandbox/ns/ipc", + UTS: "/run/sandbox/ns/uts", + }, + }, + "a path backed sandbox in the host namespaces provides none": { + spec: &runtimespec.Spec{Linux: &runtimespec.Linux{}}, + expected: customopts.PodNamespacePaths{}, + }, + "a pid namespace entry without a path declares private pid namespaces": { + spec: &runtimespec.Spec{Linux: &runtimespec.Linux{Namespaces: []runtimespec.LinuxNamespace{ + {Type: runtimespec.NetworkNamespace, Path: "/var/run/netns/cni-1"}, + {Type: runtimespec.PIDNamespace}, + }}}, + expected: customopts.PodNamespacePaths{Network: "/var/run/netns/cni-1", PrivatePID: true}, + }, + "a target container pid namespace is joined by pid": { + targetPid: 7, spec: spec, + expected: customopts.PodNamespacePaths{ + Network: "/var/run/netns/cni-1", + IPC: "/run/sandbox/ns/ipc", + UTS: "/run/sandbox/ns/uts", + PID: "/proc/7/ns/pid", + }, + }, + } { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, podNamespacePaths(tc.sandboxPid, tc.targetPid, tc.spec)) + }) + } +} + +func TestLinuxContainerMountsFromSandboxSpec(t *testing.T) { + const testSandboxID = "test-sandbox-id" + config := &runtime.ContainerConfig{ + Metadata: &runtime.ContainerMetadata{Name: "test-name", Attempt: 1}, + Linux: &runtime.LinuxContainerConfig{ + SecurityContext: &runtime.LinuxContainerSecurityContext{}, + }, + } + exists := func(paths ...string) func(string) (os.FileInfo, error) { + set := make(map[string]struct{}, len(paths)) + for _, p := range paths { + set[p] = struct{}{} + } + return func(p string) (os.FileInfo, error) { + if _, ok := set[p]; ok { + return nil, nil + } + return nil, os.ErrNotExist + } + } + sharedMount := func(containerPath, hostPath string) *runtime.Mount { + return &runtime.Mount{ContainerPath: containerPath, HostPath: hostPath, SelinuxRelabel: true} + } + + t.Run("the sandbox spec provides the pod shared files", func(t *testing.T) { + c := newTestCRIService() + c.os.(*ostesting.FakeOS).StatFn = exists("/run/sandbox/hostname", "/run/sandbox/hosts", "/run/sandbox/resolv.conf", "/run/sandbox/shm") + assert.Equal(t, []*runtime.Mount{ + sharedMount(etcHostname, "/run/sandbox/hostname"), + sharedMount(etcHosts, "/run/sandbox/hosts"), + sharedMount(resolvConfPath, "/run/sandbox/resolv.conf"), + sharedMount(devShm, "/run/sandbox/shm"), + }, c.linuxContainerMounts(testSandboxID, config, pathBackedSandboxSpec())) + }) + + t.Run("the fixed sandbox directory wins over the spec", func(t *testing.T) { + c := newTestCRIService() + c.os.(*ostesting.FakeOS).StatFn = exists(c.getSandboxHosts(testSandboxID), "/run/sandbox/hosts") + assert.Equal(t, []*runtime.Mount{ + sharedMount(etcHosts, c.getSandboxHosts(testSandboxID)), + }, c.linuxContainerMounts(testSandboxID, config, pathBackedSandboxSpec())) + }) + + t.Run("a spec source that does not exist is not mounted", func(t *testing.T) { + c := newTestCRIService() + c.os.(*ostesting.FakeOS).StatFn = exists() + assert.Empty(t, c.linuxContainerMounts(testSandboxID, config, pathBackedSandboxSpec())) + }) + + t.Run("host ipc uses the host shm", func(t *testing.T) { + c := newTestCRIService() + c.os.(*ostesting.FakeOS).StatFn = exists(devShm, "/run/sandbox/shm") + hostIPC := &runtime.ContainerConfig{ + Metadata: config.Metadata, + Linux: &runtime.LinuxContainerConfig{ + SecurityContext: &runtime.LinuxContainerSecurityContext{ + NamespaceOptions: &runtime.NamespaceOption{Ipc: runtime.NamespaceMode_NODE}, + }, + }, + } + assert.Equal(t, []*runtime.Mount{ + {ContainerPath: devShm, HostPath: devShm, SelinuxRelabel: false}, + }, c.linuxContainerMounts(testSandboxID, hostIPC, pathBackedSandboxSpec())) + }) +} diff --git a/internal/cri/server/container_create_test.go b/internal/cri/server/container_create_test.go index 227b78aeac..ff7cb0aea1 100644 --- a/internal/cri/server/container_create_test.go +++ b/internal/cri/server/container_create_test.go @@ -70,7 +70,7 @@ func TestGeneralContainerSpec(t *testing.T) { c := newTestCRIService() testSandboxID := "sandbox-id" testContainerName := "container-name" - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) } @@ -146,7 +146,7 @@ func TestPodAnnotationPassthroughContainerSpec(t *testing.T) { PodAnnotations: test.podAnnotations, } spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, - containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) assert.NotNil(t, spec) specCheck(t, testID, testSandboxID, testPid, spec) @@ -508,7 +508,7 @@ func TestContainerAnnotationPassthroughContainerSpec(t *testing.T) { ContainerAnnotations: test.containerAnnotations, } spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, - containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) assert.NotNil(t, spec) specCheck(t, testID, testSandboxID, testPid, spec) @@ -759,7 +759,7 @@ func TestLinuxContainerMounts(t *testing.T) { } c := newTestCRIService() c.os.(*ostesting.FakeOS).StatFn = test.statFn - mounts := c.linuxContainerMounts(testSandboxID, config) + mounts := c.linuxContainerMounts(testSandboxID, config, nil) assert.Equal(t, test.expectedMounts, mounts, test.desc) }) } diff --git a/internal/cri/server/container_create_windows_test.go b/internal/cri/server/container_create_windows_test.go index 2a12d16237..5cfe1401eb 100644 --- a/internal/cri/server/container_create_windows_test.go +++ b/internal/cri/server/container_create_windows_test.go @@ -165,7 +165,7 @@ func TestContainerWindowsNetworkNamespace(t *testing.T) { c := newTestCRIService() containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData() - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil, nil) assert.NoError(t, err) assert.NotNil(t, spec) specCheck(t, testID, testSandboxID, testPid, spec) @@ -187,7 +187,7 @@ func TestMountCleanPath(t *testing.T) { ContainerPath: "c:/test/container-path", HostPath: "c:/test/host-path", }) - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil, nil) assert.NoError(t, err) assert.NotNil(t, spec) specCheck(t, testID, testSandboxID, testPid, spec) @@ -207,7 +207,7 @@ func TestMountNamedPipe(t *testing.T) { ContainerPath: `\\.\pipe\foo`, HostPath: `\\.\pipe\foo`, }) - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil, nil) assert.NoError(t, err) assert.NotNil(t, spec) specCheck(t, testID, testSandboxID, testPid, spec) @@ -258,7 +258,7 @@ func TestHostProcessRequirements(t *testing.T) { sandboxConfig.Windows.SecurityContext = &runtime.WindowsSandboxSecurityContext{ HostProcess: test.sandboxHostProcess, } - _, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + _, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) if test.expectError { assert.Error(t, err) } else { @@ -355,7 +355,7 @@ func TestEntrypointAndCmdForArgsEscaped(t *testing.T) { Args: test.args, Windows: &runtime.WindowsContainerConfig{}, } - runtimeSpec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil) + runtimeSpec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil, nil) assert.NoError(t, err) assert.NotNil(t, runtimeSpec) diff --git a/internal/cri/server/sandbox_run.go b/internal/cri/server/sandbox_run.go index 1972888a7f..14bd9ec256 100644 --- a/internal/cri/server/sandbox_run.go +++ b/internal/cri/server/sandbox_run.go @@ -304,7 +304,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox } } - if err := c.sandboxService.CreateSandbox(ctx, sandboxInfo, sb.WithOptions(config), sb.WithNetNSPath(sandbox.NetNSPath)); err != nil { + if err := c.sandboxService.CreateSandbox(ctx, sandboxInfo, sb.WithOptions(config), sb.WithNetNSPath(sandbox.NetNSPath), sb.WithRuntimePath(ociRuntime.Path)); err != nil { return nil, fmt.Errorf("failed to create sandbox %q: %w", id, err) } diff --git a/plugins/sandbox/controller.go b/plugins/sandbox/controller.go index cee78ad96b..5402afe2a1 100644 --- a/plugins/sandbox/controller.go +++ b/plugins/sandbox/controller.go @@ -18,6 +18,7 @@ package sandbox import ( "context" + "errors" "fmt" "os" "time" @@ -27,6 +28,7 @@ import ( "github.com/containerd/log" "github.com/containerd/plugin" "github.com/containerd/plugin/registry" + "github.com/containerd/ttrpc" "github.com/containerd/typeurl/v2" imagespec "github.com/opencontainers/image-spec/specs-go/v1" @@ -102,10 +104,22 @@ type controllerLocal struct { var _ sandbox.Controller = (*controllerLocal)(nil) func (c *controllerLocal) cleanupShim(ctx context.Context, sandboxID string, svc runtimeAPI.TTRPCSandboxService) { - // Let the shim exit, then we can clean up the bundle after. - if _, sErr := svc.ShutdownSandbox(ctx, &runtimeAPI.ShutdownSandboxRequest{ + // The caller's context may be the reason Create or Start failed, canceled + // or past its deadline; the cleanup must still run, so it keeps the values + // of that context (namespace, logger) but not its cancellation, and each + // step gets its own deadline instead. + ctx = context.WithoutCancel(ctx) + + // Let the shim exit, then we can clean up the bundle after. The deadline + // leaves room for the shim to stop what it runs for the sandbox, which + // takes its own bounded wait. A closed connection means the shim exited + // before answering, as for Shutdown. + sctx, scancel := context.WithTimeout(ctx, 30*time.Second) + _, sErr := svc.ShutdownSandbox(sctx, &runtimeAPI.ShutdownSandboxRequest{ SandboxID: sandboxID, - }); sErr != nil { + }) + scancel() + if sErr != nil && !errors.Is(sErr, ttrpc.ErrClosed) { log.G(ctx).WithError(sErr).WithField("sandboxID", sandboxID). Error("failed to shutdown sandbox") } @@ -124,7 +138,9 @@ func (c *controllerLocal) Create(ctx context.Context, info sandbox.Sandbox, opts var coptions sandbox.CreateOptions sandboxID := info.ID for _, opt := range opts { - opt(&coptions) + if err := opt(&coptions); err != nil { + return err + } } if _, err := c.shims.Get(ctx, sandboxID); err == nil { @@ -141,10 +157,16 @@ func (c *controllerLocal) Create(ctx context.Context, info sandbox.Sandbox, opts } }() + // The shim manager accepts an absolute binary path in place of a + // runtime name. + runtimeName := info.Runtime.Name + if coptions.RuntimePath != "" { + runtimeName = coptions.RuntimePath + } shim, err := c.shims.Start(ctx, sandboxID, bundle, runtime.CreateOpts{ Spec: info.Spec, RuntimeOptions: info.Runtime.Options, - Runtime: info.Runtime.Name, + Runtime: runtimeName, TaskOptions: nil, }) if err != nil { @@ -252,8 +274,11 @@ func (c *controllerLocal) Shutdown(ctx context.Context, sandboxID string) error return err } + // A shim may exit as soon as it has shut its sandbox down, before the + // response is read; a closed connection means the shutdown happened, as + // for the task Shutdown RPC. _, err = svc.ShutdownSandbox(ctx, &runtimeAPI.ShutdownSandboxRequest{SandboxID: sandboxID}) - if err != nil { + if err != nil && !errors.Is(err, ttrpc.ErrClosed) { return fmt.Errorf("failed to shutdown sandbox: %w", errgrpc.ToNative(err)) } @@ -331,7 +356,7 @@ func (c *controllerLocal) Metrics(ctx context.Context, sandboxID string) (*types req := &runtimeAPI.SandboxMetricsRequest{SandboxID: sandboxID} resp, err := sb.SandboxMetrics(ctx, req) if err != nil { - return nil, err + return nil, errgrpc.ToNative(err) } return resp.Metrics, nil } diff --git a/plugins/sandbox/controller_test.go b/plugins/sandbox/controller_test.go new file mode 100644 index 0000000000..d702703fc9 --- /dev/null +++ b/plugins/sandbox/controller_test.go @@ -0,0 +1,35 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "context" + "testing" + + "github.com/containerd/errdefs" + "github.com/stretchr/testify/require" + + "github.com/containerd/containerd/v2/core/sandbox" +) + +// A failing create option ends Create before any shim is involved: the +// controller has no shim manager here, so getting past the options would panic. +func TestCreateOptionError(t *testing.T) { + c := &controllerLocal{} + err := c.Create(context.Background(), sandbox.Sandbox{ID: "sb"}, sandbox.WithRuntimePath("containerd-shim-example-v1")) + require.ErrorIs(t, err, errdefs.ErrInvalidArgument) +} diff --git a/script/critest.sh b/script/critest.sh index dc944a2ae9..27f8ff8af6 100755 --- a/script/critest.sh +++ b/script/critest.sh @@ -50,6 +50,17 @@ function cleanup() { trap cleanup EXIT mkdir -p ${BDIR}/{root,state} + +# SANDBOXER is the sandbox controller of the runtime handler: "podsandbox" +# (the pause container, the default) or "shim" (the Sandbox API of the shim, +# which needs no pause image). +SANDBOXER="${SANDBOXER:-podsandbox}" +SANDBOX_CONFIG="sandboxer = \"${SANDBOXER}\"" +if [ "${SANDBOXER}" = "shim" ]; then + SANDBOX_CONFIG="${SANDBOX_CONFIG} +disable_pause_image_pull = true" +fi + containerd_config="${CONTAINERD_CONFIG_FILE:-${BDIR}/config.toml}" if [[ -z "${CONTAINERD_CONFIG_FILE:-}" ]]; then cat > "${containerd_config}" < "snapshotter \"overlayfs\" doesn't support idmap mounts on this host,