From fc774e4b3c52a697d98869ccee2af6c7bff2e785 Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Thu, 15 Oct 2026 19:07:24 -0500 Subject: [PATCH] ci: add integration-runsc CI job and script/runsc/ test helpers Co-authored-by: Wei Fu Signed-off-by: Mike Brown --- .github/workflows/ci.yml | 109 +++++++++- core/metadata/containers.go | 7 +- core/metadata/containers_test.go | 38 ++++ core/metadata/sandbox.go | 12 +- core/metadata/sandbox_test.go | 37 ++++ docs/cri/config.md | 22 +- ...ntainer_cgroup_mount_options_linux_test.go | 8 + integration/container_exec_test.go | 3 + integration/container_io_leak_linux_test.go | 5 + integration/container_stats_test.go | 23 +- integration/container_stop_signal_test.go | 6 + integration/container_stop_test.go | 3 + integration/container_tty_leak_linux_test.go | 3 + .../container_update_resources_test.go | 9 + integration/image_volume_linux_test.go | 6 + integration/issue7496_linux_test.go | 3 + integration/issue7496_shutdown_linux_test.go | 9 + integration/main_test.go | 13 ++ integration/nri_linux_test.go | 6 + integration/nri_test.go | 21 ++ integration/oom_linux_test.go | 8 +- integration/pod_hostname_test.go | 3 + integration/pod_userns_linux_test.go | 9 + integration/restart_linux_test.go | 3 + integration/restart_test.go | 3 + integration/truncindex_test.go | 39 ++-- internal/cri/server/runtime_config_linux.go | 59 ++++- .../cri/server/runtime_config_linux_test.go | 204 ++++++++++++++++++ plugins/cri/images/plugin.go | 8 +- plugins/cri/images/plugin_test.go | 43 +++- script/critest.sh | 28 ++- script/runsc/config-containerd-runsc | 47 ++++ script/runsc/critest-runsc.sh | 99 +++++++++ script/runsc/install-runsc | 69 ++++++ script/runsc/run-runsc-tests.sh | 139 ++++++++++++ script/runsc/runsc-version | 1 + script/runsc/setup-runsc-test-env.sh | 80 +++++++ script/runsc/write-containerd-runsc-config.sh | 69 ++++++ script/runsc/write-runsc-shim-config.sh | 64 ++++++ script/setup/install-failpoint-binaries | 14 +- script/test/cri-integration.sh | 4 + script/vm/test-cri.sh | 3 +- 42 files changed, 1289 insertions(+), 50 deletions(-) create mode 100755 script/runsc/config-containerd-runsc create mode 100755 script/runsc/critest-runsc.sh create mode 100755 script/runsc/install-runsc create mode 100755 script/runsc/run-runsc-tests.sh create mode 100644 script/runsc/runsc-version create mode 100755 script/runsc/setup-runsc-test-env.sh create mode 100755 script/runsc/write-containerd-runsc-config.sh create mode 100755 script/runsc/write-runsc-shim-config.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b0d88908f..7d0589bfc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -693,6 +693,113 @@ jobs: LIMA_WORKDIR: / run: lima sudo dmesg -T -f kern + + integration-runsc: + name: runsc (gVisor) integration + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: [project, linters, protos, man] + + strategy: + fail-fast: false + matrix: + cgroup_driver: [systemd, cgroupfs] + + env: + CGROUP_DRIVER: ${{ matrix.cgroup_driver }} + TEST_RUNTIME: io.containerd.runsc.v1 + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/install-go + + - name: Install containerd dependencies + run: | + sudo apt-get update + sudo apt-get install -y gperf dmsetup strace xfsprogs + script/setup/install-seccomp + script/setup/install-runc + script/setup/install-cni $(grep containernetworking/plugins go.mod | awk '{print $2}') + script/setup/install-critools + script/setup/install-failpoint-binaries + + - name: Install runsc (gVisor) + run: script/runsc/install-runsc + + - name: Build containerd + env: + CGO_ENABLED: 1 + run: | + make binaries GO_BUILD_FLAGS="-mod=vendor" + sudo -E PATH=$PATH make install + + - name: CRI integration (runsc) + run: | + # Generate the runsc containerd config and pass it to cri-integration + # so utils.sh skips its own config generation. + cfg=$(mktemp /tmp/containerd-config-runsc-XXXXXX.toml) + trap 'rm -f "$cfg"' EXIT + script/runsc/write-runsc-shim-config.sh "$CGROUP_DRIVER" systrap + script/runsc/write-containerd-runsc-config.sh "$cfg" "$CGROUP_DRIVER" + # The external config and runsc shim config already contain the cgroup settings. + # Keep CGROUP_DRIVER set but empty so utils.sh does not append runc.options again. + CGROUP_DRIVER= CONTAINERD_CONFIG_FILE="$cfg" RUNTIME=runsc RUNTIME_TYPE=io.containerd.runsc.v1 make cri-integration + + - name: Runsc memory baseline + run: | + set -eux + free -h + cat /proc/meminfo | grep -E 'MemAvailable|SwapFree|SwapTotal' + systemctl show containerd --property=MemoryMax --property=MemoryHigh || true + sudo systemd-cgtop -b -n 1 || true + + - name: critest (runsc) + env: + GOMAXPROCS: "2" + GINKGO_NODES: "2" + GOMEMLIMIT: "4GiB" + run: | + mkdir -p "${{github.workspace}}/report" + cfg=$(mktemp /tmp/containerd-config-runsc-XXXXXX.toml) + trap 'rm -f "$cfg"' EXIT + script/runsc/write-containerd-runsc-config.sh "$cfg" "$CGROUP_DRIVER" + sudo -E \ + CGROUP_DRIVER="$CGROUP_DRIVER" \ + CONTAINERD_CONFIG_FILE="$cfg" \ + GOMAXPROCS="$GOMAXPROCS" \ + GOMEMLIMIT="$GOMEMLIMIT" \ + GINKGO_NODES="$GINKGO_NODES" \ + PATH="$PATH" \ + script/runsc/critest-runsc.sh "${{github.workspace}}/report" + + - name: Host Status + if: always() + run: | + free -h + df -h + ps aux --sort=-%mem | head -n 30 + sudo systemctl status containerd --no-pager || true + sudo journalctl -k --no-pager -n 200 || true + sudo dmesg -T | grep -iE 'oom|out of memory|killed process' || true + + - name: Kernel Message + if: failure() + run: | + sudo dmesg -T | tail -n 200 || true + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: TestResults-runsc-${{ matrix.cgroup_driver }} + path: | + *-junit.xml + *-gotest.json + ${{github.workspace}}/report/*.xml + ${{github.workspace}}/report/*.log + + tests-cri-in-userns: name: "CRI-in-UserNS" @@ -796,7 +903,7 @@ jobs: name: Report required job statuses runs-on: ubuntu-24.04 # List job dependencies which are required to pass status checks in order to be merged via merge queue. - needs: [linters, project, protos, binaries, integration-linux, integration-windows, tests-cri-in-userns, tests-mac-os, node-e2e] + needs: [linters, project, protos, binaries, integration-linux, integration-windows, integration-runsc, tests-cri-in-userns, tests-mac-os, node-e2e] if: ${{ always() }} steps: - run: exit 1 diff --git a/core/metadata/containers.go b/core/metadata/containers.go index 9d72b5086f..1a15a3a1ef 100644 --- a/core/metadata/containers.go +++ b/core/metadata/containers.go @@ -383,7 +383,12 @@ func readContainer(container *containers.Container, bkt *bolt.Bucket) error { if err != nil { return err } - container.Runtime.Options = o + // Avoid assigning a typed-nil (*anypb.Any)(nil) to the typeurl.Any + // interface, which would produce a non-nil interface value that + // fails type-URL lookup. + if o != nil { + container.Runtime.Options = o + } case string(bucketKeySpec): var spec types.Any if err := proto.Unmarshal(v, &spec); err != nil { diff --git a/core/metadata/containers_test.go b/core/metadata/containers_test.go index 077bf84f26..994eaa9483 100644 --- a/core/metadata/containers_test.go +++ b/core/metadata/containers_test.go @@ -721,3 +721,41 @@ func testEnv(t *testing.T) (context.Context, *bolt.DB) { return ctx, db } + +// TestContainerNilRuntimeOptionsRoundTrip verifies that a container stored +// without Runtime.Options round-trips back with a genuinely nil interface +// value, not a typed-nil *types.Any wrapped in a typeurl.Any interface. +// +// checkContainersEqual uses compareNil, which treats a typed-nil pointer +// wrapped in an interface as equal to a true nil interface and therefore would +// not catch this regression. A non-nil interface with a nil underlying pointer +// causes type-URL lookup failures at runtime. +func TestContainerNilRuntimeOptionsRoundTrip(t *testing.T) { + ctx, db := testEnv(t) + store := NewContainerStore(NewDB(db, nil, nil)) + + spec := &specs.Spec{} + encoded, err := typeurl.MarshalAnyToProto(spec) + require.NoError(t, err) + + in := containers.Container{ + ID: "nil-runtime-opts", + Spec: encoded, + Runtime: containers.RuntimeInfo{ + Name: "testruntime", + // Options intentionally left nil. + }, + } + + _, err = store.Create(ctx, in) + require.NoError(t, err) + + out, err := store.Get(ctx, "nil-runtime-opts") + require.NoError(t, err) + + // Use direct interface comparison — not compareNil — to distinguish a + // true nil interface from a typed-nil pointer wrapped in an interface. + if out.Runtime.Options != nil { + t.Errorf("Runtime.Options: want nil interface, got %T(%v)", out.Runtime.Options, out.Runtime.Options) + } +} diff --git a/core/metadata/sandbox.go b/core/metadata/sandbox.go index c97010a006..bf20c38b21 100644 --- a/core/metadata/sandbox.go +++ b/core/metadata/sandbox.go @@ -371,10 +371,15 @@ func (s *sandboxStore) read(parent *bbolt.Bucket, id []byte) (api.Sandbox, error return api.Sandbox{}, err } - inst.Spec, err = boltutil.ReadAny(bucket, bucketKeySpec) + spec, err := boltutil.ReadAny(bucket, bucketKeySpec) if err != nil { return api.Sandbox{}, err } + // Avoid assigning a typed-nil (*anypb.Any)(nil) to the typeurl.Any interface, + // which would produce a non-nil interface value that fails type-URL lookup. + if spec != nil { + inst.Spec = spec + } runtimeBucket := bucket.Bucket(bucketKeyRuntime) if runtimeBucket == nil { @@ -382,10 +387,13 @@ func (s *sandboxStore) read(parent *bbolt.Bucket, id []byte) (api.Sandbox, error } inst.Runtime.Name = string(runtimeBucket.Get(bucketKeyName)) - inst.Runtime.Options, err = boltutil.ReadAny(runtimeBucket, bucketKeyOptions) + runtimeOpts, err := boltutil.ReadAny(runtimeBucket, bucketKeyOptions) if err != nil { return api.Sandbox{}, err } + if runtimeOpts != nil { + inst.Runtime.Options = runtimeOpts + } inst.Extensions, err = boltutil.ReadExtensions(bucket) if err != nil { diff --git a/core/metadata/sandbox_test.go b/core/metadata/sandbox_test.go index 94cd428bd3..93bb7d5bc4 100644 --- a/core/metadata/sandbox_test.go +++ b/core/metadata/sandbox_test.go @@ -264,3 +264,40 @@ func assertEqualInstances(t *testing.T, x, y api.Sandbox) { t.Fatalf("x and y are different: %s", diff) } } + +// TestSandboxNilInterfaceRoundTrip verifies that a sandbox stored without a +// Spec or Runtime.Options round-trips back with genuinely nil interface values. +// +// compareNil (used by assertEqualInstances) treats a typed-nil pointer wrapped +// in an interface as equal to a true nil interface, so it would not catch a +// regression where ReadAny returns (*types.Any)(nil) and that typed-nil is +// stored directly in the typeurl.Any field — producing a non-nil interface +// that fails type-URL lookup at runtime. +func TestSandboxNilInterfaceRoundTrip(t *testing.T) { + ctx, db := testDB(t) + store := NewSandboxStore(db) + + in := api.Sandbox{ + ID: "nil-fields", + Runtime: api.RuntimeOpts{Name: "test"}, + // Spec and Runtime.Options are intentionally left nil. + } + + if _, err := store.Create(ctx, in); err != nil { + t.Fatal(err) + } + + out, err := store.Get(ctx, "nil-fields") + if err != nil { + t.Fatal(err) + } + + // Use direct interface comparison — not compareNil — to distinguish a + // true nil interface from a typed-nil pointer wrapped in an interface. + if out.Spec != nil { + t.Errorf("Spec: want nil interface, got %T(%v)", out.Spec, out.Spec) + } + if out.Runtime.Options != nil { + t.Errorf("Runtime.Options: want nil interface, got %T(%v)", out.Runtime.Options, out.Runtime.Options) + } +} diff --git a/docs/cri/config.md b/docs/cri/config.md index 3ba0381eb1..a9dc456314 100644 --- a/docs/cri/config.md +++ b/docs/cri/config.md @@ -57,12 +57,22 @@ kubeadm users should also see [the kubeadm documentation](https://kubernetes.io/ > runtime and the `KubeletConfiguration` configuration step above is not > needed. > -> When determining the cgroup driver, containerd uses the `SystemdCgroup` -> setting from runc-based runtime classes, starting from the default runtime -> class. If no runc-based runtime classes have been configured containerd -> relies on auto-detection based on determining if systemd is running. -> Note that all runc-based runtime classes should be configured to have the -> same `SystemdCgroup` setting in order to avoid unexpected behavior. +> When determining the cgroup driver, containerd inspects the runtime classes +> in order, starting from the default runtime class: +> +> - For runc-based runtime classes (`io.containerd.runc.v2`), containerd reads +> the `SystemdCgroup` field from the runtime's `options` block. +> - For generic shims that use `runtimeoptions/v1` (e.g. `io.containerd.runsc.v1` +> for gVisor), containerd reads the cgroup driver from the shim's config file +> (`ConfigPath`) or inline config body (`ConfigBody`). It first looks for a +> top-level `SystemdCgroup` key (boolean), then for +> `[runsc_config] systemd-cgroup` which the runsc shim stores as a TOML string +> (`"true"` or `"false"`). +> - If no runtime class provides cgroup driver information, containerd falls back +> to auto-detection based on whether systemd is running. +> +> Note that all runtime classes should be configured with the same cgroup driver +> setting in order to avoid unexpected behavior. > > The automatic cgroup driver configuration for kubelet feature is supported in > containerd v2.0 and later. diff --git a/integration/container_cgroup_mount_options_linux_test.go b/integration/container_cgroup_mount_options_linux_test.go index 7ffb0bf662..62a3ed52a6 100644 --- a/integration/container_cgroup_mount_options_linux_test.go +++ b/integration/container_cgroup_mount_options_linux_test.go @@ -28,6 +28,14 @@ import ( ) func TestPrivilegedContainerCgroupMountOptions(t *testing.T) { + if runtimeHandlerIsRunsc() { + // This test verifies that the host /sys/fs/cgroup mount options + // (nsdelegate, memory_recursiveprot) are preserved after running a + // privileged container — a runc-specific cgroup namespace concern. + // gVisor containers never modify the host cgroup mount, so the test + // is not applicable under runsc. + t.Skip("host cgroup mount option test is not applicable to runsc") + } if cgroups.Mode() != cgroups.Unified { t.Skip("Requires cgroup v2") } diff --git a/integration/container_exec_test.go b/integration/container_exec_test.go index 9c15387204..32216164ee 100644 --- a/integration/container_exec_test.go +++ b/integration/container_exec_test.go @@ -28,6 +28,9 @@ import ( ) func TestContainerDrainExecIOAfterExit(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not support draining exec IO for background processes") + } // FIXME(fuweid): support it for windows container. if runtime.GOOS == "windows" { t.Skip("it seems that windows platform doesn't support detached process. skip it") diff --git a/integration/container_io_leak_linux_test.go b/integration/container_io_leak_linux_test.go index 9b7240c3f5..d1d0fd64e8 100644 --- a/integration/container_io_leak_linux_test.go +++ b/integration/container_io_leak_linux_test.go @@ -33,6 +33,11 @@ func TestContainerIOLeakAfterStartFailed(t *testing.T) { if f := os.Getenv("RUNC_FLAVOR"); f != "" && f != "runc" { t.Skip("test requires runc") } + if runtimeHandlerIsRunsc() { + // getShimPid connects to the per-sandbox shim ttrpc socket, which is + // a runc-shim implementation detail not available under runsc. + t.Skip("getShimPid introspection is runc-shim-specific, not applicable to runsc") + } t.Log("Create a sandbox") sb, sbConfig := PodSandboxConfigWithCleanup(t, "sandbox", "container-io-leak-after-start-failed") testImage := images.Get(images.BusyBox) diff --git a/integration/container_stats_test.go b/integration/container_stats_test.go index 021400ad6b..14a1b52496 100644 --- a/integration/container_stats_test.go +++ b/integration/container_stats_test.go @@ -31,6 +31,9 @@ import ( // Test to verify for a container ID func TestContainerStats(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not expose per-container CPU nanosecond stats") + } t.Logf("Create a pod config and run sandbox container") sb, sbConfig := PodSandboxConfigWithCleanup(t, "sandbox1", "stats") @@ -73,6 +76,9 @@ func TestContainerStats(t *testing.T) { // Test to verify if the consumed stats are correct. func TestContainerConsumedStats(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not expose per-container memory working-set stats") + } t.Logf("Create a pod config and run sandbox container") sb, sbConfig := PodSandboxConfigWithCleanup(t, "sandbox1", "stats") @@ -129,7 +135,7 @@ func TestContainerConsumedStats(t *testing.T) { } go func() { - _, _, err = runtimeService.ExecSync(cn, command, 30*time.Second) + _, _, _ = runtimeService.ExecSync(cn, command, 30*time.Second) }() require.NoError(t, Eventually(func() (bool, error) { @@ -146,6 +152,9 @@ func TestContainerConsumedStats(t *testing.T) { // Test to verify filtering without any filter func TestContainerListStats(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not expose per-container CPU nanosecond stats") + } var ( stats []*runtime.ContainerStats err error @@ -201,6 +210,9 @@ func TestContainerListStats(t *testing.T) { // Test to verify filtering given a specific container ID // TODO Convert the filter tests into table driven tests and unit tests func TestContainerListStatsWithIdFilter(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not expose per-container CPU nanosecond stats") + } var ( stats []*runtime.ContainerStats err error @@ -261,6 +273,9 @@ func TestContainerListStatsWithIdFilter(t *testing.T) { // Test to verify filtering given a specific Sandbox ID. Stats for // all the containers in a pod should be returned func TestContainerListStatsWithSandboxIdFilter(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not expose per-container CPU nanosecond stats") + } var ( stats []*runtime.ContainerStats err error @@ -322,6 +337,9 @@ func TestContainerListStatsWithSandboxIdFilter(t *testing.T) { // Test to verify filtering given a specific container ID and // sandbox ID func TestContainerListStatsWithIdSandboxIdFilter(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not expose per-container CPU nanosecond stats") + } var ( stats []*runtime.ContainerStats err error @@ -430,6 +448,9 @@ func testStats(t *testing.T, } func TestContainerSysfsStatsWithPrivilegedPod(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("Privileged containers are not supported by runsc") + } if goruntime.GOOS == "windows" { t.Skip("Doesn't care about filesystem properties on windows") } diff --git a/integration/container_stop_signal_test.go b/integration/container_stop_signal_test.go index 1044466575..4eedda07bd 100644 --- a/integration/container_stop_signal_test.go +++ b/integration/container_stop_signal_test.go @@ -33,6 +33,9 @@ func TestContainerStopSignals(t *testing.T) { if goruntime.GOOS == "windows" { t.Skip("Skipped on Windows.") } + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not support host network") + } testCases := []struct { name string @@ -130,6 +133,9 @@ func TestDefaultContainerStopSignal(t *testing.T) { if goruntime.GOOS == "windows" { t.Skip("Skipped on Windows.") } + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not support host network") + } t.Logf("Create a pod config") sb, sbConfig := PodSandboxConfigWithCleanup(t, "sandbox", "default-stop-signal", WithHostNetwork) diff --git a/integration/container_stop_test.go b/integration/container_stop_test.go index b73d6fe847..76c904a3f7 100644 --- a/integration/container_stop_test.go +++ b/integration/container_stop_test.go @@ -34,6 +34,9 @@ func TestSharedPidMultiProcessContainerStop(t *testing.T) { "podpid": PodSandboxConfig("sandbox", "pod-pid-container-stop", WithPodPid), } { t.Run(name, func(t *testing.T) { + if name == "hostpid" && runtimeHandlerIsRunsc() { + t.Skip("runsc does not support host PID namespace") + } t.Log("Create a shared pid sandbox") sb, err := runtimeService.RunPodSandbox(sbConfig, *runtimeHandler) require.NoError(t, err) diff --git a/integration/container_tty_leak_linux_test.go b/integration/container_tty_leak_linux_test.go index 67e8759c78..c41acb5553 100644 --- a/integration/container_tty_leak_linux_test.go +++ b/integration/container_tty_leak_linux_test.go @@ -35,6 +35,9 @@ import ( ) func TestContainerTTYLeakAfterExit(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc uses grouped shims; direct shim socket connection is not supported") + } t.Log("Create a sandbox") sb, sbConfig := PodSandboxConfigWithCleanup(t, "sandbox", "container-tty-leak-after-exit") diff --git a/integration/container_update_resources_test.go b/integration/container_update_resources_test.go index 8eb6cc0c28..a473a6d9fb 100644 --- a/integration/container_update_resources_test.go +++ b/integration/container_update_resources_test.go @@ -154,6 +154,9 @@ func isSwapLikelyEnabled() bool { } func TestUpdateContainerResources_MemorySwap(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not implement cgroup memory swap limits") + } if !isSwapLikelyEnabled() { t.Skipf("Swap or swap accounting are not enabled. Swap is required for this test") return @@ -226,6 +229,9 @@ func TestUpdateContainerResources_MemorySwap(t *testing.T) { } func TestUpdateContainerResources_MemoryLimit(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not implement cgroup memory limits") + } t.Log("Create a sandbox") sb, sbConfig := PodSandboxConfigWithCleanup(t, "sandbox", "update-container-resources") @@ -306,6 +312,9 @@ func TestUpdateContainerResources_MemoryLimit(t *testing.T) { } func TestUpdateContainerResources_StatusUpdated(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not implement cgroup memory limits") + } t.Log("Create a sandbox") sb, sbConfig := PodSandboxConfigWithCleanup(t, "sandbox", "update-container-resources") diff --git a/integration/image_volume_linux_test.go b/integration/image_volume_linux_test.go index d4a556ebd9..d16a5cd88d 100644 --- a/integration/image_volume_linux_test.go +++ b/integration/image_volume_linux_test.go @@ -134,6 +134,9 @@ func TestImageVolumeBasic(t *testing.T) { if !selinux.GetEnabled() { t.Skip("SELinux is not enabled") } + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not support SELinux labels in the OCI spec") + } } podCtx, cnID, err := setupRunningContainerWithImageVolume(t, tc.selinuxLevel, tc.containerImage, tc.imageVolumeImage, tc.imageSubPath, tc.containerPath) @@ -358,6 +361,9 @@ func TestImageVolumeSetupIfContainerdRestarts(t *testing.T) { } func TestImageVolumeWithUserNamespace(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not support host user namespaces") + } // Check if user namespace and idmap are supported if !supportsUserNS() { t.Skip("user namespace not supported") diff --git a/integration/issue7496_linux_test.go b/integration/issue7496_linux_test.go index 0e42e85b11..9e96a11a0d 100644 --- a/integration/issue7496_linux_test.go +++ b/integration/issue7496_linux_test.go @@ -43,6 +43,9 @@ import ( // // NOTE: https://github.com/containerd/containerd/issues/8931 is the same issue. func TestIssue7496(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not use host umount2; strace injection does not apply") + } ctx := namespaces.WithNamespace(context.Background(), "k8s.io") t.Logf("Create a pod config and run sandbox container") diff --git a/integration/issue7496_shutdown_linux_test.go b/integration/issue7496_shutdown_linux_test.go index 765a41cba1..ac3fe83e00 100644 --- a/integration/issue7496_shutdown_linux_test.go +++ b/integration/issue7496_shutdown_linux_test.go @@ -104,6 +104,9 @@ func TestIssue7496_ShouldRetryShutdown(t *testing.T) { } func TestKillShimPublishesTaskExitAndDeleteEvents(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc uses grouped shims; direct shim socket connection is not supported") + } ctx := namespaces.WithNamespace(t.Context(), "k8s.io") sbConfig := PodSandboxConfig("sandbox", t.Name(), WithHostNetwork) @@ -156,6 +159,9 @@ func TestKillShimPublishesTaskExitAndDeleteEvents(t *testing.T) { // Regression test for https://github.com/containerd/containerd/issues/13293 (orphaned shim state after shim SIGKILL). // This test asserts that containerd publishes task exit and delete events for a container when the shim disconnects. func TestKillShimPublishesTaskExitAndDeleteEventsForContainer(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc uses grouped shims; direct shim socket connection is not supported") + } ctx := namespaces.WithNamespace(t.Context(), "k8s.io") sbConfig := PodSandboxConfig("sandbox", t.Name(), WithHostNetwork) @@ -429,6 +435,9 @@ func countTaskExitDeleteEventsUntilBarrier(t *testing.T, id, barrierTopic string } func TestShutdownShimWhenPauseExitsBeforeWorkload(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc uses grouped shims; direct shim socket connection is not supported") + } ctx := namespaces.WithNamespace(t.Context(), "k8s.io") t.Logf("RunPodSandbox") diff --git a/integration/main_test.go b/integration/main_test.go index 7d1f5ab813..07dfa8a224 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -70,9 +70,22 @@ var ( var criEndpoint = flag.String("cri-endpoint", "unix:///run/containerd/containerd.sock", "The endpoint of cri plugin.") var runtimeHandler = flag.String("runtime-handler", "", "The runtime handler to use in the test.") +var runtimeType = flag.String("runtime-type", "", "The OCI runtime type of the default runtime handler (e.g. io.containerd.runsc.v1).") var containerdBin = flag.String("containerd-bin", "containerd", "The containerd binary name. The name is used to restart containerd during test.") var buildDir = flag.String("build-dir", "", "Build output directory for containerd binaries") +func runtimeHandlerIsRunsc() bool { + if *runtimeType == "io.containerd.runsc.v1" { + return true + } + // Match documented handler name aliases for gVisor / runsc. + switch *runtimeHandler { + case "runsc", "gvisor": + return true + } + return strings.HasPrefix(*runtimeHandler, "runsc-") +} + func TestMain(m *testing.M) { flag.Parse() if err := ConnectDaemons(); err != nil { diff --git a/integration/nri_linux_test.go b/integration/nri_linux_test.go index 1178854b22..4752f48cb8 100644 --- a/integration/nri_linux_test.go +++ b/integration/nri_linux_test.go @@ -33,6 +33,9 @@ import ( // Test NRI networking capabilities. func TestNriPluginNetworkingSynchronization(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc uses its own netstack; NRI cannot read pod IPs from the host network namespace") + } skipNriTestIfNecessary(t) t.Log("Test that NRI plugins get pod networking attributes on synchronization.") @@ -107,6 +110,9 @@ func TestNriPluginNetworkingSynchronization(t *testing.T) { } func TestNriPluginNetworkingLifecycle(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc uses its own netstack; NRI cannot read pod IPs from the host network namespace") + } skipNriTestIfNecessary(t) t.Log("Test that NRI plugins get pod networking attributes during the pod lifecycle.") diff --git a/integration/nri_test.go b/integration/nri_test.go index fc9160bb45..981cc63b02 100644 --- a/integration/nri_test.go +++ b/integration/nri_test.go @@ -303,6 +303,9 @@ func TestNriLinuxDeviceInjection(t *testing.T) { // Test linux cpuset adjustment by NRI plugins. func TestNriLinuxCpusetAdjustment(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not propagate NRI cpuset adjustments to the container's /proc/self/status") + } skipNriTestIfNecessary(t, map[string]bool{ "not enough online CPUs for test": len(getAvailableCpuset(t)) < 2, @@ -348,6 +351,9 @@ func TestNriLinuxCpusetAdjustment(t *testing.T) { // Test linux memset adjustment by NRI plugins. func TestNriLinuxMemsetAdjustment(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not propagate NRI memset adjustments to the container's /proc/self/status") + } skipNriTestIfNecessary(t, map[string]bool{ "not enough online memory nodes for test": len(getAvailableMemset(t)) < 2, @@ -393,6 +399,9 @@ func TestNriLinuxMemsetAdjustment(t *testing.T) { // Test creation-time linux cpuset update of existing containers by NRI plugins. func TestNriLinuxCpusetAdjustmentUpdate(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not propagate NRI cpuset adjustments to the container's /proc/self/status") + } skipNriTestIfNecessary(t, map[string]bool{ "not enough online CPUs for test": len(getAvailableCpuset(t)) < 2, @@ -464,6 +473,9 @@ func TestNriLinuxCpusetAdjustmentUpdate(t *testing.T) { // Test creation-time linux memset update of existing containers by NRI plugins. func TestNriLinuxMemsetAdjustmentUpdate(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not propagate NRI memset adjustments to the container's /proc/self/status") + } skipNriTestIfNecessary(t, map[string]bool{ "not enough online memory nodes for test": len(getAvailableMemset(t)) < 2, @@ -532,6 +544,9 @@ func TestNriLinuxMemsetAdjustmentUpdate(t *testing.T) { // Test pod sandbox updates received by NRI plugins. func TestNriUpdatePodSandbox(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not implement cgroup memory limits") + } skipNriTestIfNecessary(t) t.Log("Test that NRI plugins receive pod sandbox updates.") @@ -574,6 +589,9 @@ func TestNriUpdatePodSandbox(t *testing.T) { // Test pod sandbox resource updates persist across containerd restarts. func TestUpdatePodSandboxWithRestart(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc sandboxes do not survive containerd restart; state recovery is not supported") + } skipNriTestIfNecessary(t) t.Log("Test that pod sandbox resource updates persist across containerd restarts.") @@ -613,6 +631,9 @@ func TestUpdatePodSandboxWithRestart(t *testing.T) { // Test NRI vs. containerd restart. func TestNriPluginContainerdRestart(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc sandboxes do not survive containerd restart; state recovery is not supported") + } skipNriTestIfNecessary(t) t.Log("Test NRI plugins vs. containerd restart.") diff --git a/integration/oom_linux_test.go b/integration/oom_linux_test.go index 60b0d2ec0c..86a35f0304 100644 --- a/integration/oom_linux_test.go +++ b/integration/oom_linux_test.go @@ -28,11 +28,13 @@ import ( "github.com/containerd/containerd/v2/integration/images" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - criruntime "k8s.io/cri-api/pkg/apis/runtime/v1" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" ) func TestOOMEventMonitor(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not implement cgroup memory limits; OOMKilled reporting requires host cgroup enforcement") + } workDir := t.TempDir() rawCfg := fmt.Sprintf(` @@ -96,7 +98,7 @@ version = 3 wg.Go(func() { cntrID := podCtx.createContainer(fmt.Sprintf("round-%d-running", round), busyboxImage, - criruntime.ContainerState_CONTAINER_RUNNING, + runtimeapi.ContainerState_CONTAINER_RUNNING, WithCommand( "sh", "-c", @@ -117,7 +119,7 @@ version = 3 } state := status.GetState() - if state != criruntime.ContainerState_CONTAINER_EXITED { + if state != runtimeapi.ContainerState_CONTAINER_EXITED { return false, nil } diff --git a/integration/pod_hostname_test.go b/integration/pod_hostname_test.go index 661362b026..e55547b3c6 100644 --- a/integration/pod_hostname_test.go +++ b/integration/pod_hostname_test.go @@ -65,6 +65,9 @@ func TestPodHostname(t *testing.T) { if test.needsHostNetwork && goruntime.GOOS == "windows" { t.Skip("Skipped on Windows.") } + if test.needsHostNetwork && runtimeHandlerIsRunsc() { + t.Skip("runsc does not support host network") + } testPodLogDir := t.TempDir() opts := append(test.opts, WithPodLogDirectory(testPodLogDir)) diff --git a/integration/pod_userns_linux_test.go b/integration/pod_userns_linux_test.go index 11f059db84..27409d9f1c 100644 --- a/integration/pod_userns_linux_test.go +++ b/integration/pod_userns_linux_test.go @@ -127,6 +127,9 @@ func traversePath(tPath string) error { } func TestPodUserNS(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not support host user namespaces") + } containerID := uint32(0) hostID := uint32(65536) size := uint32(65536) @@ -350,6 +353,9 @@ func TestPodUserNS(t *testing.T) { // [1]: https://github.com/containerd/containerd/issues/10598 // [2]: https://github.com/anakryiko/retsnoop func TestIssue10598(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not support host user namespaces") + } if !supportsUserNS() { t.Skip("User namespaces are not supported") } @@ -447,6 +453,9 @@ func TestIssue10598(t *testing.T) { // // https://github.com/containerd/containerd/issues/11852 func TestUsernsVolumeCopyUp(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc does not support host user namespaces") + } if !supportsUserNS() { t.Skip("User namespaces are not supported") } diff --git a/integration/restart_linux_test.go b/integration/restart_linux_test.go index abec034c0b..42185495b4 100644 --- a/integration/restart_linux_test.go +++ b/integration/restart_linux_test.go @@ -30,6 +30,9 @@ import ( ) func TestContainerdRestartSandboxRecover(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc sandboxes do not survive containerd restart; state recovery is not supported") + } sbStatuses := map[string]runtime.PodSandboxState{ // Sandbox with unknown status will be NotReady when returned from ListPodSandbox "sandbox_unknown": runtime.PodSandboxState_SANDBOX_NOTREADY, diff --git a/integration/restart_test.go b/integration/restart_test.go index e7a8a424a4..ffcddf7f98 100644 --- a/integration/restart_test.go +++ b/integration/restart_test.go @@ -34,6 +34,9 @@ import ( // Restart test must run sequentially. func TestContainerdRestart(t *testing.T) { + if runtimeHandlerIsRunsc() { + t.Skip("runsc sandboxes do not survive containerd restart; state recovery is not supported") + } type container struct { name string id string diff --git a/integration/truncindex_test.go b/integration/truncindex_test.go index a3264471cd..4144eab6d6 100644 --- a/integration/truncindex_test.go +++ b/integration/truncindex_test.go @@ -73,8 +73,11 @@ func TestTruncIndex(t *testing.T) { assert.Equal(t, sb, sbStatus.Id) t.Logf("Forward port for sandbox by truncindex") - _, err = runtimeService.PortForward(&runtimeapi.PortForwardRequest{PodSandboxId: sbTruncIndex, Port: []int32{80}}) - assert.NoError(t, err) + if !runtimeHandlerIsRunsc() { + // runsc does not support port-forwarding into its own network namespace. + _, err = runtimeService.PortForward(&runtimeapi.PortForwardRequest{PodSandboxId: sbTruncIndex, Port: []int32{80}}) + assert.NoError(t, err) + } // TODO(yanxuean): add test case for ListPodSandbox @@ -108,21 +111,27 @@ func TestTruncIndex(t *testing.T) { }() t.Logf("Stats the container") - cStats, err := runtimeService.ContainerStats(cnTruncIndex) - require.NoError(t, err) - assert.Equal(t, cn, cStats.Attributes.Id) + if !runtimeHandlerIsRunsc() { + // runsc does not expose per-container stats. + cStats, err := runtimeService.ContainerStats(cnTruncIndex) + require.NoError(t, err) + assert.Equal(t, cn, cStats.Attributes.Id) + } t.Logf("Update container memory limit after started") - if goruntime.GOOS != "windows" { - err = runtimeService.UpdateContainerResources(cnTruncIndex, &runtimeapi.LinuxContainerResources{ - MemoryLimitInBytes: 50 * 1024 * 1024, - }, nil) - assert.NoError(t, err) - } else { - err = runtimeService.UpdateContainerResources(cnTruncIndex, nil, &runtimeapi.WindowsContainerResources{ - MemoryLimitInBytes: 50 * 1024 * 1024, - }) - assert.NoError(t, err) + if !runtimeHandlerIsRunsc() { + // runsc does not implement cgroup memory limits. + if goruntime.GOOS != "windows" { + err = runtimeService.UpdateContainerResources(cnTruncIndex, &runtimeapi.LinuxContainerResources{ + MemoryLimitInBytes: 50 * 1024 * 1024, + }, nil) + assert.NoError(t, err) + } else { + err = runtimeService.UpdateContainerResources(cnTruncIndex, nil, &runtimeapi.WindowsContainerResources{ + MemoryLimitInBytes: 50 * 1024 * 1024, + }) + assert.NoError(t, err) + } } t.Logf("Execute cmd in container") diff --git a/internal/cri/server/runtime_config_linux.go b/internal/cri/server/runtime_config_linux.go index de4bf192d8..a199f7d9cb 100644 --- a/internal/cri/server/runtime_config_linux.go +++ b/internal/cri/server/runtime_config_linux.go @@ -18,12 +18,16 @@ package server import ( "context" + "os" "sort" + "strconv" runcoptions "github.com/containerd/containerd/api/types/runc/options" + runtimeoptions "github.com/containerd/containerd/api/types/runtimeoptions/v1" criconfig "github.com/containerd/containerd/v2/internal/cri/config" "github.com/containerd/containerd/v2/internal/cri/systemd" "github.com/containerd/log" + "github.com/pelletier/go-toml/v2" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) @@ -72,11 +76,60 @@ func (c *criService) getCgroupDriver(ctx context.Context) runtime.CgroupDriver { func getCgroupDriverFromRuntimeHandlerOpts(opts any) (runtime.CgroupDriver, bool) { switch v := opts.(type) { case *runcoptions.Options: - systemdCgroup := v.SystemdCgroup - if systemdCgroup { + if v.SystemdCgroup { return runtime.CgroupDriver_SYSTEMD, true } return runtime.CgroupDriver_CGROUPFS, true + case *runtimeoptions.Options: + // Generic shims (e.g. io.containerd.runsc.v1) carry their config + // via a path to a config file or inline via ConfigBody. + // We look for a top-level SystemdCgroup key or [runsc_config].systemd-cgroup. + // Only return a driver when the key is explicitly present. + var data []byte + var err error + if v.ConfigPath != "" { + data, err = os.ReadFile(v.ConfigPath) + } else { + data = v.ConfigBody + } + if err != nil || len(data) == 0 { + return runtime.CgroupDriver_CGROUPFS, false + } + + var cfg struct { + SystemdCgroup *bool `toml:"SystemdCgroup"` + RunscConfig map[string]any `toml:"runsc_config"` + } + if err := toml.Unmarshal(data, &cfg); err != nil { + return runtime.CgroupDriver_CGROUPFS, false + } + + if cfg.SystemdCgroup != nil { + if *cfg.SystemdCgroup { + return runtime.CgroupDriver_SYSTEMD, true + } + return runtime.CgroupDriver_CGROUPFS, true + } + + // The runsc shim stores the cgroup driver under [runsc_config].systemd-cgroup. + // containerd-shim-runsc-v1 expects this value as a TOML string ("true"/"false") + // because its config struct maps the field to a Go string; accept both forms. + if val, ok := cfg.RunscConfig["systemd-cgroup"]; ok { + switch v := val.(type) { + case bool: + if v { + return runtime.CgroupDriver_SYSTEMD, true + } + return runtime.CgroupDriver_CGROUPFS, true + case string: + if b, err := strconv.ParseBool(v); err == nil { + if b { + return runtime.CgroupDriver_SYSTEMD, true + } + return runtime.CgroupDriver_CGROUPFS, true + } + } + } } - return runtime.CgroupDriver_SYSTEMD, false + return runtime.CgroupDriver_CGROUPFS, false } diff --git a/internal/cri/server/runtime_config_linux_test.go b/internal/cri/server/runtime_config_linux_test.go index 50546f4aa8..be3c6de349 100644 --- a/internal/cri/server/runtime_config_linux_test.go +++ b/internal/cri/server/runtime_config_linux_test.go @@ -18,6 +18,8 @@ package server import ( "context" + "os" + "path/filepath" "testing" criconfig "github.com/containerd/containerd/v2/internal/cri/config" @@ -38,6 +40,56 @@ func newFakeRuntimeConfig(runcV2, systemdCgroup bool) criconfig.Runtime { return r } +// newFakeGenericConfigPathRuntime returns a runtime config for a generic shim +// that points ConfigPath to a temporary config file with SystemdCgroup set. +func newFakeGenericConfigPathRuntime(t *testing.T, runtimeType string, systemdCgroup bool) criconfig.Runtime { + t.Helper() + val := "false" + if systemdCgroup { + val = "true" + } + content := "SystemdCgroup = " + val + "\n" + f := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(f, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return criconfig.Runtime{Type: runtimeType, Options: map[string]any{ + "ConfigPath": f, + }} +} + +// newFakeRunscConfigPathRuntime returns a runtime config for the runsc shim +// that uses config_path to point at a temporary shim config file. The file +// uses the [runsc_config] section format that containerd-shim-runsc-v1 reads. +// tomlString controls whether systemd-cgroup is written as a TOML string +// ("true"/"false") — as produced by write-runsc-shim-config.sh — or as a +// bare TOML boolean (true/false). +func newFakeRunscConfigPathRuntime(t *testing.T, runtimeType string, systemdCgroup bool, tomlString bool) criconfig.Runtime { + t.Helper() + var val string + if tomlString { + if systemdCgroup { + val = `"true"` + } else { + val = `"false"` + } + } else { + if systemdCgroup { + val = "true" + } else { + val = "false" + } + } + content := "[runsc_config]\n systemd-cgroup = " + val + "\n" + f := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(f, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return criconfig.Runtime{Type: runtimeType, Options: map[string]any{ + "ConfigPath": f, + }} +} + func TestRuntimeConfig(t *testing.T) { autoDetected := runtime.CgroupDriver_CGROUPFS if systemd.IsRunningSystemd() { @@ -90,6 +142,64 @@ func TestRuntimeConfig(t *testing.T) { }, expectedCgroupDriver: runtime.CgroupDriver_SYSTEMD, }, + { + // A generic shim with non-empty options and no ConfigPath must + // fall through to host auto-detection, not be silently treated as cgroupfs. + desc: "generic shim without ConfigPath falls through to auto-detect", + defaultRuntime: "other", + runtimes: map[string]criconfig.Runtime{ + "other": {Type: "io.containerd.other.v1", Options: map[string]any{"SomeOtherKey": "value"}}, + }, + expectedCgroupDriver: autoDetected, + }, + { + desc: "generic shim with inline SystemdCgroup false", + defaultRuntime: "generic", + runtimes: map[string]criconfig.Runtime{ + "generic": {Type: "io.containerd.generic.v1", Options: map[string]any{"SystemdCgroup": false}}, + }, + expectedCgroupDriver: runtime.CgroupDriver_CGROUPFS, + }, + { + desc: "generic shim with inline SystemdCgroup true", + defaultRuntime: "generic", + runtimes: map[string]criconfig.Runtime{ + "generic": {Type: "io.containerd.generic.v1", Options: map[string]any{"SystemdCgroup": true}}, + }, + expectedCgroupDriver: runtime.CgroupDriver_SYSTEMD, + }, + { + desc: "runsc shim with inline runsc_config systemd-cgroup bool true", + defaultRuntime: "runsc", + runtimes: map[string]criconfig.Runtime{ + "runsc": {Type: "io.containerd.runsc.v1", Options: map[string]any{"runsc_config": map[string]any{"systemd-cgroup": true}}}, + }, + expectedCgroupDriver: runtime.CgroupDriver_SYSTEMD, + }, + { + desc: "runsc shim with inline runsc_config systemd-cgroup bool false", + defaultRuntime: "runsc", + runtimes: map[string]criconfig.Runtime{ + "runsc": {Type: "io.containerd.runsc.v1", Options: map[string]any{"runsc_config": map[string]any{"systemd-cgroup": false}}}, + }, + expectedCgroupDriver: runtime.CgroupDriver_CGROUPFS, + }, + { + desc: "runsc shim with inline runsc_config systemd-cgroup string true", + defaultRuntime: "runsc", + runtimes: map[string]criconfig.Runtime{ + "runsc": {Type: "io.containerd.runsc.v1", Options: map[string]any{"runsc_config": map[string]any{"systemd-cgroup": "true"}}}, + }, + expectedCgroupDriver: runtime.CgroupDriver_SYSTEMD, + }, + { + desc: "runsc shim with inline runsc_config systemd-cgroup string false", + defaultRuntime: "runsc", + runtimes: map[string]criconfig.Runtime{ + "runsc": {Type: "io.containerd.runsc.v1", Options: map[string]any{"runsc_config": map[string]any{"systemd-cgroup": "false"}}}, + }, + expectedCgroupDriver: runtime.CgroupDriver_CGROUPFS, + }, } { t.Run(test.desc, func(t *testing.T) { c := newTestCRIService() @@ -101,4 +211,98 @@ func TestRuntimeConfig(t *testing.T) { assert.Equal(t, test.expectedCgroupDriver, resp.Linux.CgroupDriver, "got unexpected cgroup driver") }) } + + // ConfigPath cases use temp files so cannot be in the table above. + t.Run("generic config_path, cgroupfs", func(t *testing.T) { + c := newTestCRIService() + c.config.RuntimeConfig.ContainerdConfig.DefaultRuntimeName = "generic" + c.config.RuntimeConfig.ContainerdConfig.Runtimes = map[string]criconfig.Runtime{ + "generic": newFakeGenericConfigPathRuntime(t, "io.containerd.generic.v1", false), + } + resp, err := c.RuntimeConfig(context.TODO(), &runtime.RuntimeConfigRequest{}) + assert.NoError(t, err) + assert.Equal(t, runtime.CgroupDriver_CGROUPFS, resp.Linux.CgroupDriver, "got unexpected cgroup driver") + }) + + t.Run("generic config_path, systemd", func(t *testing.T) { + c := newTestCRIService() + c.config.RuntimeConfig.ContainerdConfig.DefaultRuntimeName = "generic" + c.config.RuntimeConfig.ContainerdConfig.Runtimes = map[string]criconfig.Runtime{ + "generic": newFakeGenericConfigPathRuntime(t, "io.containerd.generic.v1", true), + } + resp, err := c.RuntimeConfig(context.TODO(), &runtime.RuntimeConfigRequest{}) + assert.NoError(t, err) + assert.Equal(t, runtime.CgroupDriver_SYSTEMD, resp.Linux.CgroupDriver, "got unexpected cgroup driver") + }) + + t.Run("generic config_path default overrides runc", func(t *testing.T) { + c := newTestCRIService() + c.config.RuntimeConfig.ContainerdConfig.DefaultRuntimeName = "generic" + c.config.RuntimeConfig.ContainerdConfig.Runtimes = map[string]criconfig.Runtime{ + "runc": newFakeRuntimeConfig(true, true), + "generic": newFakeGenericConfigPathRuntime(t, "io.containerd.generic.v1", false), + } + resp, err := c.RuntimeConfig(context.TODO(), &runtime.RuntimeConfigRequest{}) + assert.NoError(t, err) + assert.Equal(t, runtime.CgroupDriver_CGROUPFS, resp.Linux.CgroupDriver, "got unexpected cgroup driver") + }) + // Test both the bare-boolean form and the TOML-string form ("true"/"false") + // produced by write-runsc-shim-config.sh, since the shim config struct maps + // systemd-cgroup to a Go string. + t.Run("runsc config_path, cgroupfs (bool)", func(t *testing.T) { + c := newTestCRIService() + c.config.RuntimeConfig.ContainerdConfig.DefaultRuntimeName = "runsc" + c.config.RuntimeConfig.ContainerdConfig.Runtimes = map[string]criconfig.Runtime{ + "runsc": newFakeRunscConfigPathRuntime(t, "io.containerd.runsc.v1", false, false), + } + resp, err := c.RuntimeConfig(context.TODO(), &runtime.RuntimeConfigRequest{}) + assert.NoError(t, err) + assert.Equal(t, runtime.CgroupDriver_CGROUPFS, resp.Linux.CgroupDriver, "got unexpected cgroup driver") + }) + + t.Run("runsc config_path, systemd (bool)", func(t *testing.T) { + c := newTestCRIService() + c.config.RuntimeConfig.ContainerdConfig.DefaultRuntimeName = "runsc" + c.config.RuntimeConfig.ContainerdConfig.Runtimes = map[string]criconfig.Runtime{ + "runsc": newFakeRunscConfigPathRuntime(t, "io.containerd.runsc.v1", true, false), + } + resp, err := c.RuntimeConfig(context.TODO(), &runtime.RuntimeConfigRequest{}) + assert.NoError(t, err) + assert.Equal(t, runtime.CgroupDriver_SYSTEMD, resp.Linux.CgroupDriver, "got unexpected cgroup driver") + }) + + t.Run("runsc config_path, cgroupfs (string)", func(t *testing.T) { + c := newTestCRIService() + c.config.RuntimeConfig.ContainerdConfig.DefaultRuntimeName = "runsc" + c.config.RuntimeConfig.ContainerdConfig.Runtimes = map[string]criconfig.Runtime{ + "runsc": newFakeRunscConfigPathRuntime(t, "io.containerd.runsc.v1", false, true), + } + resp, err := c.RuntimeConfig(context.TODO(), &runtime.RuntimeConfigRequest{}) + assert.NoError(t, err) + assert.Equal(t, runtime.CgroupDriver_CGROUPFS, resp.Linux.CgroupDriver, "got unexpected cgroup driver") + }) + + t.Run("runsc config_path, systemd (string)", func(t *testing.T) { + c := newTestCRIService() + c.config.RuntimeConfig.ContainerdConfig.DefaultRuntimeName = "runsc" + c.config.RuntimeConfig.ContainerdConfig.Runtimes = map[string]criconfig.Runtime{ + "runsc": newFakeRunscConfigPathRuntime(t, "io.containerd.runsc.v1", true, true), + } + resp, err := c.RuntimeConfig(context.TODO(), &runtime.RuntimeConfigRequest{}) + assert.NoError(t, err) + assert.Equal(t, runtime.CgroupDriver_SYSTEMD, resp.Linux.CgroupDriver, "got unexpected cgroup driver") + }) + + t.Run("runsc config_path missing file falls through to auto-detect", func(t *testing.T) { + c := newTestCRIService() + c.config.RuntimeConfig.ContainerdConfig.DefaultRuntimeName = "runsc" + c.config.RuntimeConfig.ContainerdConfig.Runtimes = map[string]criconfig.Runtime{ + "runsc": {Type: "io.containerd.runsc.v1", Options: map[string]any{ + "ConfigPath": "/nonexistent/runsc/config.toml", + }}, + } + resp, err := c.RuntimeConfig(context.TODO(), &runtime.RuntimeConfigRequest{}) + assert.NoError(t, err) + assert.Equal(t, autoDetected, resp.Linux.CgroupDriver, "got unexpected cgroup driver") + }) } diff --git a/plugins/cri/images/plugin.go b/plugins/cri/images/plugin.go index f263bed437..c767ce574e 100644 --- a/plugins/cri/images/plugin.go +++ b/plugins/cri/images/plugin.go @@ -238,22 +238,22 @@ func migrateConfig(dst, src map[string]any) { } var runtimePlatforms map[string]any - if v, ok := dst["runtime_platform"]; ok { + if v, ok := dst["runtime_platforms"]; ok { runtimePlatforms = v.(map[string]any) } else { runtimePlatforms = map[string]any{} } for runtime, v := range runtimesConf.(map[string]any) { runtimeConf := v.(map[string]any) - if snapshotter, ok := runtimeConf["snapshot"]; ok && snapshotter != "" { + if snapshotter, ok := runtimeConf["snapshotter"]; ok && snapshotter != "" { runtimePlatforms[runtime] = map[string]any{ - "platform": platforms.DefaultStrict(), + "platform": platforms.DefaultString(), "snapshotter": snapshotter, } } } if len(runtimePlatforms) > 0 { - dst["runtime_platform"] = runtimePlatforms + dst["runtime_platforms"] = runtimePlatforms } for _, key := range []string{ diff --git a/plugins/cri/images/plugin_test.go b/plugins/cri/images/plugin_test.go index 3845f51b62..bcb9ee37d6 100644 --- a/plugins/cri/images/plugin_test.go +++ b/plugins/cri/images/plugin_test.go @@ -20,9 +20,11 @@ import ( "context" "testing" - "github.com/containerd/containerd/v2/plugins" + "github.com/containerd/platforms" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/containerd/containerd/v2/plugins" ) func TestSandboxImageConfigMigration(t *testing.T) { @@ -68,3 +70,42 @@ func TestRegistryConfigMigration(t *testing.T) { configPath := v.(string) assert.Equal(t, path, configPath) } + +func TestRuntimePlatformsConfigMigration(t *testing.T) { + snapshotter := "devmapper" + grpcCri := map[string]any{ + "containerd": map[string]any{ + "runtimes": map[string]any{ + "kata": map[string]any{ + // Old per-runtime snapshotter key is "snapshotter", not "snapshot". + "snapshotter": snapshotter, + }, + // Runtime with no snapshotter should not appear in runtime_platforms. + "runc": map[string]any{}, + }, + }, + } + pluginConfigs := map[string]any{ + string(plugins.GRPCPlugin) + ".cri": grpcCri, + } + configMigration(context.Background(), 2, pluginConfigs) + + v, ok := pluginConfigs[string(plugins.CRIServicePlugin)+".images"] + require.True(t, ok) + images := v.(map[string]any) + + // The destination key is "runtime_platforms" (plural), matching the TOML + // tag on ImageConfig.RuntimePlatforms. + v, ok = images["runtime_platforms"] + require.True(t, ok, "runtime_platforms key must be present after migration") + runtimePlatforms := v.(map[string]any) + + kata, ok := runtimePlatforms["kata"] + require.True(t, ok, "kata runtime must be present in runtime_platforms") + kataConf := kata.(map[string]any) + assert.Equal(t, platforms.DefaultString(), kataConf["platform"]) + assert.Equal(t, snapshotter, kataConf["snapshotter"]) + + _, ok = runtimePlatforms["runc"] + assert.False(t, ok, "runc has no snapshotter and must not appear in runtime_platforms") +} diff --git a/script/critest.sh b/script/critest.sh index 5f785a19ae..c810750f82 100755 --- a/script/critest.sh +++ b/script/critest.sh @@ -50,7 +50,9 @@ function cleanup() { trap cleanup EXIT mkdir -p ${BDIR}/{root,state} -cat > ${BDIR}/config.toml < "${containerd_config}" <> ${BDIR}/config.toml <> "${containerd_config}" <&2 + exit 1 fi GINKGO_SKIP_TEST=() @@ -120,15 +126,21 @@ ls /etc/cni/net.d /usr/local/bin/containerd \ -a ${BDIR}/c.sock \ - --config ${BDIR}/config.toml \ + --config "${containerd_config}" \ --root ${BDIR}/root \ --state ${BDIR}/state \ --log-level debug &> "$report_dir/containerd.log" & # Make sure containerd is ready before calling critest. -for i in $(seq 1 10) -do - crictl --runtime-endpoint ${BDIR}/c.sock info && break || sleep 1 +for i in $(seq 1 10); do + crictl --runtime-endpoint ${BDIR}/c.sock info 2>/dev/null && break + if [[ "${i}" -eq 10 ]]; then + echo "containerd did not become ready after 10 seconds" >&2 + exit 1 + fi + sleep 1 done -critest --report-dir "$report_dir" --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8 "${GINKGO_SKIP_TEST[@]}" "${GINKGO_FOCUS_TEST[@]}" ${EXTRA_CRITEST_OPTIONS:-} +ginkgo_procs="${GINKGO_NODES:-8}" + +critest --report-dir "$report_dir" --runtime-endpoint=unix:///${BDIR}/c.sock --parallel="${ginkgo_procs}" "${GINKGO_SKIP_TEST[@]}" "${GINKGO_FOCUS_TEST[@]}" ${EXTRA_CRITEST_OPTIONS:-} diff --git a/script/runsc/config-containerd-runsc b/script/runsc/config-containerd-runsc new file mode 100755 index 0000000000..205e810eb0 --- /dev/null +++ b/script/runsc/config-containerd-runsc @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Writes a complete /etc/containerd/config.toml for a runsc (gVisor) setup, +# replacing any existing config. This avoids fragile in-place patching of the +# file written by script/setup/config-containerd. +# +# Required environment variable: +# CGROUP_DRIVER — must be either "cgroupfs" or "systemd" +# +set -eux -o pipefail + +: "${CGROUP_DRIVER:?CGROUP_DRIVER must be set to cgroupfs or systemd}" + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1; pwd -P)" + +SUDO='' +if [[ "$(id -u)" != "0" ]]; then + SUDO='sudo' +fi + +${SUDO} mkdir -p /etc/containerd + +# Delegate to the shared helper so the config is defined in one place. +tmpfile="$(mktemp /tmp/containerd-config-runsc-XXXXXX.toml)" +trap 'rm -f "${tmpfile}"' EXIT +"${script_dir}/write-containerd-runsc-config.sh" "${tmpfile}" "${CGROUP_DRIVER}" +${SUDO} cp "${tmpfile}" /etc/containerd/config.toml + +# Write the runsc shim config file. write-runsc-shim-config.sh removes the +# higher-precedence /run/containerd/runsc/config.toml before writing to /etc, +# so stale files from previous runs cannot override the requested settings. +"${script_dir}/write-runsc-shim-config.sh" "${CGROUP_DRIVER}" diff --git a/script/runsc/critest-runsc.sh b/script/runsc/critest-runsc.sh new file mode 100755 index 0000000000..0cba3253fd --- /dev/null +++ b/script/runsc/critest-runsc.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Runs critest against a containerd instance configured for runsc (gVisor). +# Skips tests for features that gVisor does not support. +# +# Usage: critest-runsc.sh +# +# Required environment variable: +# CGROUP_DRIVER — must be either "cgroupfs" or "systemd" +# +# The containerd and runsc shim configs must already be written by the caller. +# Pass the containerd config through CONTAINERD_CONFIG_FILE. +# +set -eu -o pipefail + +report_dir=${1:?report_dir argument is required} + +: "${CGROUP_DRIVER:?CGROUP_DRIVER must be set to cgroupfs or systemd}" +: "${CONTAINERD_CONFIG_FILE:?CONTAINERD_CONFIG_FILE must point to the runsc containerd config}" + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1; pwd -P)" + +# Fail fast if a runsc sandbox remains before critest. The root directory itself +# may remain because runsc keeps its shared null network namespace there. +_runsc_root=/run/containerd-runsc-test +if [[ -e "${_runsc_root}" ]]; then + if ! _runsc_ids="$(runsc --root="${_runsc_root}" list --quiet)"; then + echo "ERROR: unable to inspect runsc root ${_runsc_root}" >&2 + exit 1 + fi + if [[ -n "${_runsc_ids}" ]]; then + echo "ERROR: stale runsc sandboxes found in ${_runsc_root}:" >&2 + echo "${_runsc_ids}" >&2 + exit 1 + fi +fi + +# Skip tests for features gVisor does not support. +# See script/vm/test-cri.sh for the full rationale. +skip_tests=( + 'HostIpc is true' + 'HostNetwork is true' + 'HostPID' + 'SELinux' + 'should support safe sysctls' + 'should support unsafe sysctls' + 'mount propagation' + 'rshared' + 'non-recursive readonly' + 'adding capability' + 'MaskedPaths' + 'ReadonlyPaths' + 'SeccompProfilePath' + 'UserNamespaces' + 'portforward' + 'listing pod sandbox metrics' + 'listing container stats|listing stats for' + 'should enforce a apparmor_profile blocking writes' + 'should allow privilege escalation when false' + 'runtime should support Privileged is true' +) +# Skip OOM tests for runsc entirely - it intentionally triggers container OOM +# kills which causes host memory pressure on CI runners (both cgroupfs and systemd). +skip_tests+=( + 'should terminate with exitCode 137' + 'OOMKilled' +) +skip_arg=$(IFS='|'; echo "${skip_tests[*]}") + +if [[ -n "${SKIP_TEST:-}" ]]; then + SKIP_TEST="${SKIP_TEST}|${skip_arg}" +else + SKIP_TEST="${skip_arg}" +fi +export SKIP_TEST + +if [[ -z "${GINKGO_NODES:-}" ]]; then + # Cap default parallelism to avoid host resource exhaustion on high-core runners. + _procs=$(($(nproc) + 2)) + GINKGO_NODES=$(( _procs > 8 ? 8 : _procs )) +fi +export GINKGO_NODES + +exec "${script_dir}/../critest.sh" "${report_dir}" diff --git a/script/runsc/install-runsc b/script/runsc/install-runsc new file mode 100755 index 0000000000..a86efc95f1 --- /dev/null +++ b/script/runsc/install-runsc @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Downloads and installs runsc (gVisor) and containerd-shim-runsc-v1 +# from the official gVisor GCS release bucket. +# The version is read from script/runsc/runsc-version or overridden via $RUNSC_VERSION. +# +set -eu -o pipefail + +script_dir="$(cd -- "$(dirname -- "$0")" > /dev/null 2>&1; pwd -P)" + +# e2e and Cirrus will fail with "sudo: command not found" +SUDO='' +if [[ "$(id -u)" != "0" ]]; then + SUDO='sudo' +fi + +: "${RUNSC_VERSION:=$(cat "${script_dir}/runsc-version")}" +: "${RUNSC_REPO:=https://storage.googleapis.com/gvisor/releases/release}" + +# GCS uses Linux arch names (x86_64/aarch64), not Go arch names (amd64/arm64) +case "$(uname -m)" in + x86_64|aarch64) ARCH="$(uname -m)" ;; + amd64) ARCH="x86_64" ;; + arm64) ARCH="aarch64" ;; + *) echo >&2 "unsupported arch for runsc: $(uname -m)"; exit 1 ;; +esac + +# Download the release tarball (includes runsc, containerd-shim-runsc-v1, and +# the required gvisor-bin/ sidecar executables) and verify its checksum. +tarball="$(mktemp /tmp/gvisor-XXXXXX.tar.bz2)" +tmpdir="$(mktemp -d /tmp/gvisor-XXXXXX)" +trap 'rm -rf "${tmpdir}" "${tarball}"' EXIT + +curl -fsSL --retry 5 -o "${tarball}" \ + "${RUNSC_REPO}/${RUNSC_VERSION}/${ARCH}/gvisor.tar.bz2" + +# Verify the sha512 checksum published alongside the tarball. +expected_sha512="$(curl -fsSL --retry 5 "${RUNSC_REPO}/${RUNSC_VERSION}/${ARCH}/gvisor.tar.bz2.sha512" | awk '{print $1}')" +actual_sha512="$(sha512sum "${tarball}" | awk '{print $1}')" +if [[ "${actual_sha512}" != "${expected_sha512}" ]]; then + echo >&2 "sha512 mismatch for gvisor.tar.bz2: expected ${expected_sha512}, got ${actual_sha512}" + exit 1 +fi + +# Extract into a temporary directory, then install. +tar -C "${tmpdir}" -xjf "${tarball}" + +$SUDO install -m 0755 "${tmpdir}/runsc" /usr/local/bin/runsc +$SUDO install -m 0755 "${tmpdir}/containerd-shim-runsc-v1" /usr/local/bin/containerd-shim-runsc-v1 + +# Install the gvisor-bin/ sidecar executables that runsc requires at runtime. +# runsc looks for gvisor-bin/ next to its own executable. +$SUDO mkdir -p /usr/local/bin/gvisor-bin +$SUDO install -m 0755 "${tmpdir}"/gvisor-bin/* /usr/local/bin/gvisor-bin/ diff --git a/script/runsc/run-runsc-tests.sh b/script/runsc/run-runsc-tests.sh new file mode 100755 index 0000000000..71b1593b45 --- /dev/null +++ b/script/runsc/run-runsc-tests.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Runs the integration-runsc CI test suite locally against an environment +# provisioned by setup-runsc-test-env.sh. +# +# Usage: sudo CGROUP_DRIVER=systemd script/runsc/run-runsc-tests.sh +# +# Optional: set FOCUS to run a subset of cri-integration tests, e.g. +# sudo FOCUS=TestContainerPrivileged ... run-runsc-tests.sh +# +set -eux -o pipefail + +if [[ "$(id -u)" != "0" ]]; then + echo "must be run as root (use sudo)" >&2 + exit 1 +fi + +: "${CGROUP_DRIVER:=systemd}" +: "${FOCUS:=}" + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1; pwd -P)" +containerd_dir="$(cd -- "${script_dir}/../.." > /dev/null 2>&1; pwd -P)" + +export GOPATH="${GOPATH:-/go}" +export PATH="/usr/local/go/bin:${GOPATH}/bin:/usr/local/bin:/usr/local/sbin:${PATH}" + +report_dir="${containerd_dir}/report-runsc-${CGROUP_DRIVER}" +mkdir -p "${report_dir}" + +cd "${containerd_dir}" + +# keep in sync with write-runsc-shim-config.sh +_runsc_root=/run/containerd-runsc-test +# write-runsc-shim-config.sh removes /run/containerd/runsc/config.toml (the +# higher-precedence shim config path) before writing the test config to /etc. +# Save both config paths now so the EXIT trap can restore them. +_runsc_runtime_cfg=/run/containerd/runsc/config.toml +_runsc_runtime_cfg_saved= +if [[ -f "${_runsc_runtime_cfg}" ]]; then + _runsc_runtime_cfg_saved="$(mktemp /tmp/containerd-runsc-runtime-cfg-XXXXXX.toml)" + cp "${_runsc_runtime_cfg}" "${_runsc_runtime_cfg_saved}" +fi + +_runsc_etc_cfg=/etc/containerd/runsc/config.toml +_runsc_etc_cfg_saved= +if [[ -f "${_runsc_etc_cfg}" ]]; then + _runsc_etc_cfg_saved="$(mktemp /tmp/containerd-runsc-etc-cfg-XXXXXX.toml)" + cp "${_runsc_etc_cfg}" "${_runsc_etc_cfg_saved}" +fi + +# _cleanup_runsc_root kills any surviving shims and runsc workers that hold +# mounts under _runsc_root busy, then removes the directory. Safe to call +# multiple times; all operations are best-effort (|| true). +_cleanup_runsc_root() { + if command -v runsc >/dev/null 2>&1; then + local _ids + _ids="$(runsc --root="${_runsc_root}" list --quiet 2>/dev/null || true)" + local _id + for _id in ${_ids}; do + runsc --root="${_runsc_root}" delete --force "${_id}" 2>/dev/null || true + done + fi + # Kill shims before removing the root; they can hold state mounts busy. + while IFS= read -r _pid; do + [[ -z "${_pid}" ]] && continue + kill -KILL "${_pid}" 2>/dev/null || true + done < <( + pgrep -f "containerd-shim-runsc-v1.*containerd-test" || true + pgrep -f "runsc.*--root=${_runsc_root}" || true + ) + # null-netns is a network-namespace bind-mount that findmnt may not + # enumerate; unmount and kill holders explicitly before the retry loop. + umount -l "${_runsc_root}/null-netns" 2>/dev/null || true + fuser -km "${_runsc_root}/null-netns" 2>/dev/null || true + local _i + for _i in {1..20}; do + if command -v findmnt >/dev/null 2>&1; then + findmnt -R -n -o TARGET "${_runsc_root}" 2>/dev/null | sort -r | + while IFS= read -r _mount; do + [[ -z "${_mount}" ]] && continue + umount -l "${_mount}" 2>/dev/null || true + done + fi + rm -rf "${_runsc_root:?}" && return + sleep 1 + done +} + +cfg=$(mktemp /tmp/containerd-config-runsc-XXXXXX.toml) +# On exit: restore the pre-existing shim configs (if any were saved), +# remove configs written by write-runsc-shim-config.sh when none existed, +# clean up the runsc root, and remove the temp containerd config. +trap ' + if [[ -n "${_runsc_runtime_cfg_saved}" ]]; then + mv "${_runsc_runtime_cfg_saved}" "${_runsc_runtime_cfg}" 2>/dev/null || true + fi + if [[ -n "${_runsc_etc_cfg_saved}" ]]; then + mv "${_runsc_etc_cfg_saved}" "${_runsc_etc_cfg}" 2>/dev/null || true + else + rm -f "${_runsc_etc_cfg}" 2>/dev/null || true + fi + _cleanup_runsc_root + rm -f "${cfg}" +' EXIT + +"${script_dir}/write-runsc-shim-config.sh" "${CGROUP_DRIVER}" systrap +"${script_dir}/write-containerd-runsc-config.sh" "${cfg}" "${CGROUP_DRIVER}" + +# The generated config and runsc shim config already contain the cgroup settings. +# Keep CGROUP_DRIVER set but empty so utils.sh does not append runc.options again. +CGROUP_DRIVER= \ +FOCUS="${FOCUS}" \ +CONTAINERD_CONFIG_FILE="${cfg}" \ +RUNTIME=runsc \ +RUNTIME_TYPE=io.containerd.runsc.v1 \ + make cri-integration + +echo "==> critest (CGROUP_DRIVER=${CGROUP_DRIVER})" +CGROUP_DRIVER="${CGROUP_DRIVER}" \ + CONTAINERD_CONFIG_FILE="${cfg}" \ + "${script_dir}/critest-runsc.sh" "${report_dir}" + +echo "" +echo "All tests passed. Reports in: ${report_dir}" diff --git a/script/runsc/runsc-version b/script/runsc/runsc-version new file mode 100644 index 0000000000..d640185822 --- /dev/null +++ b/script/runsc/runsc-version @@ -0,0 +1 @@ +20260817.0 diff --git a/script/runsc/setup-runsc-test-env.sh b/script/runsc/setup-runsc-test-env.sh new file mode 100755 index 0000000000..a53a89926b --- /dev/null +++ b/script/runsc/setup-runsc-test-env.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Provisions the local machine to run the integration-runsc CI job locally. +# After running this setup script, use run-runsc-tests.sh to execute the test suite. +# +# Usage: sudo CGROUP_DRIVER=systemd script/runsc/setup-runsc-test-env.sh +# +set -eux -o pipefail + +if [[ "$(id -u)" != "0" ]]; then + echo "must be run as root (use sudo)" >&2 + exit 1 +fi + +: "${CGROUP_DRIVER:=systemd}" + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1; pwd -P)" +setup_dir="$(cd -- "${script_dir}/../setup" > /dev/null 2>&1; pwd -P)" +containerd_dir="$(cd -- "${script_dir}/../.." > /dev/null 2>&1; pwd -P)" + +export GOPATH="${GOPATH:-/go}" +export PATH="/usr/local/go/bin:${GOPATH}/bin:/usr/local/bin:/usr/local/sbin:${PATH}" + +cd "${containerd_dir}" + +echo "==> apt packages" +apt-get update -q +apt-get install -y gperf dmsetup strace xfsprogs + +echo "==> libseccomp" +"${setup_dir}/install-seccomp" + +echo "==> runc" +RUNC_FLAVOR=runc "${setup_dir}/install-runc" +runc --version + +echo "==> CNI plugins" +"${setup_dir}/install-cni" "$(grep containernetworking/plugins go.mod | awk '{print $2}')" + +echo "==> critools (critest + crictl)" +"${setup_dir}/install-critools" +critest --version +crictl --version + +echo "==> runsc (gVisor)" +"${script_dir}/install-runsc" +runsc --version +containerd-shim-runsc-v1 -v + +echo "==> build containerd + failpoint binaries" +make BUILDTAGS="seccomp no_btrfs no_devmapper no_zfs" binaries +make install +# failpoint shim + CNI plugin + loopback-v2 needed by several cri-integration tests +"${setup_dir}/install-failpoint-binaries" + +echo "==> erofs (kernel module + userspace tools)" +modprobe erofs || true +apt-get install -y erofs-utils || true # best-effort; tests skip gracefully if unavailable + +echo "==> configure containerd for runsc (CGROUP_DRIVER=${CGROUP_DRIVER})" +CGROUP_DRIVER="${CGROUP_DRIVER}" "${script_dir}/config-containerd-runsc" + +echo "" +echo "Setup complete. Run tests with:" +echo " sudo CGROUP_DRIVER=${CGROUP_DRIVER} script/runsc/run-runsc-tests.sh" diff --git a/script/runsc/write-containerd-runsc-config.sh b/script/runsc/write-containerd-runsc-config.sh new file mode 100755 index 0000000000..fdec38959b --- /dev/null +++ b/script/runsc/write-containerd-runsc-config.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Writes a containerd config.toml for use with runsc (gVisor) into the path +# given as the first argument. The second argument controls the cgroup driver: +# pass "systemd" to enable the systemd driver, anything else selects cgroupfs. +# +# Usage: write-containerd-runsc-config.sh +# +# The runsc shim config (including platform and systemd-cgroup) must already +# exist at /etc/containerd/runsc/config.toml before containerd is started. +# Use write-runsc-shim-config.sh to write it. +set -eu -o pipefail + +output=${1:?output path argument is required} +cgroup_driver=${2:?cgroup_driver argument is required: pass "systemd" or "cgroupfs"} +# Normalize to a bare TOML boolean: accept "true"/"systemd" → true, anything else → false. +case "${cgroup_driver}" in + true|systemd) systemd_cgroup=true ;; + *) systemd_cgroup=false ;; +esac + +cat > "${output}" < [] +# +# pass "systemd" to enable the systemd cgroup driver; +# anything else selects cgroupfs. +# optional runsc platform (e.g. "systrap", "kvm"). +# Omit to let runsc use its compiled-in default. +# Use "systrap" on hosts without KVM support (CI runners). +# +# The shim config sets root to a test-specific path (/run/containerd-runsc-test) +# so sandbox state is isolated from the shared default (/run/containerd/runsc). +# Callers that clean up after the test should remove /run/containerd-runsc-test. +# +set -eu -o pipefail + +# Keep in sync with run-runsc-tests.sh and critest-runsc.sh. +_runsc_test_root=/run/containerd-runsc-test + +SUDO='' +if [[ "$(id -u)" != "0" ]]; then + SUDO='sudo' +fi + +case "${1:-false}" in + true|systemd) systemd_cgroup='"true"' ;; + *) systemd_cgroup='"false"' ;; +esac + +platform="${2:-}" + +# Remove the higher-precedence runtime path to avoid stale overrides. +${SUDO} rm -f /run/containerd/runsc/config.toml + +${SUDO} mkdir -p /etc/containerd/runsc + +{ + echo "" + echo "[runsc_config]" + echo " systemd-cgroup = ${systemd_cgroup}" + echo " root = \"${_runsc_test_root}\"" + if [[ -n "${platform}" ]]; then + echo " platform = \"${platform}\"" + fi +} | ${SUDO} tee /etc/containerd/runsc/config.toml > /dev/null diff --git a/script/setup/install-failpoint-binaries b/script/setup/install-failpoint-binaries index 0ade0fca67..4cdc1f5b77 100755 --- a/script/setup/install-failpoint-binaries +++ b/script/setup/install-failpoint-binaries @@ -21,6 +21,12 @@ # set -euo pipefail +# Some environments (e2e, Cirrus) run this script as root without sudo available. +SUDO='' +if [[ "$(id -u)" != "0" ]]; then + SUDO='sudo' +fi + base_dir="$(dirname "${BASH_SOURCE[0]}")" root_dir="$( cd "${base_dir}" && pwd )"/../.. @@ -28,16 +34,16 @@ cd "${root_dir}" CNI_BIN_DIR=${CNI_BIN_DIR:-"/opt/cni/bin"} make bin/cni-bridge-fp -sudo install bin/cni-bridge-fp "${CNI_BIN_DIR}" +${SUDO} install bin/cni-bridge-fp "${CNI_BIN_DIR}" SHIM_BIN_DIR=${SHIM_BIN_DIR:-"/usr/local/bin"} make bin/containerd-shim-runc-fp-v1 -sudo install bin/containerd-shim-runc-fp-v1 "${SHIM_BIN_DIR}" +${SUDO} install bin/containerd-shim-runc-fp-v1 "${SHIM_BIN_DIR}" RUNCFP_BIN_DIR=${RUNCFP_BIN_DIR:-"/usr/local/bin"} make bin/runc-fp -sudo install bin/runc-fp "${RUNCFP_BIN_DIR}" +${SUDO} install bin/runc-fp "${RUNCFP_BIN_DIR}" LOOPBACKV2_BIN_DIR=${LOOPBACKV2_BIN_DIR:-"/usr/local/bin"} make bin/loopback-v2 -sudo install bin/loopback-v2 "${LOOPBACKV2_BIN_DIR}" +${SUDO} install bin/loopback-v2 "${LOOPBACKV2_BIN_DIR}" diff --git a/script/test/cri-integration.sh b/script/test/cri-integration.sh index 70a1a146ed..c8abca5f23 100755 --- a/script/test/cri-integration.sh +++ b/script/test/cri-integration.sh @@ -37,6 +37,9 @@ else fi # RUNTIME is the runtime handler to use in the test. RUNTIME=${RUNTIME:-""} +# RUNTIME_TYPE is the OCI runtime type for the default handler (e.g. io.containerd.runsc.v1). +# Passed to --runtime-type so that runtimeHandlerIsRunsc() detects runsc regardless of handler name. +RUNTIME_TYPE=${RUNTIME_TYPE:-""} mkdir -p "${REPORT_DIR}" test_setup "${REPORT_DIR}" @@ -55,6 +58,7 @@ CMD+="${PWD}/bin/cri-integration.test${EXE_SUFFIX}" ${CMD} --test.run="${FOCUS}" --test.v \ --cri-endpoint="${CONTAINERD_SOCK}" \ --runtime-handler="${RUNTIME}" \ + --runtime-type="${RUNTIME_TYPE}" \ --containerd-bin="${CONTAINERD_BIN}" \ --build-dir="${ROOT}/bin" \ --image-list="${TEST_IMAGE_LIST:-}" "@" && test_exit_code=$? || test_exit_code=$? diff --git a/script/vm/test-cri.sh b/script/vm/test-cri.sh index ae5fec3376..f2176cf97d 100755 --- a/script/vm/test-cri.sh +++ b/script/vm/test-cri.sh @@ -64,7 +64,8 @@ skip_test_args=$( IFS='|' echo "${skip_tests[*]}" ) -critest_args=(--parallel=$(($(nproc) + 2)) --ginkgo.skip="${skip_test_args}") +ginkgo_procs="${GINKGO_NODES:-$(($(nproc) + 2))}" +critest_args=(--parallel="${ginkgo_procs}" --ginkgo.skip="${skip_test_args}") if [[ -n $REPORT_DIR ]]; then mkdir -p "${REPORT_DIR}" critest_args+=(--report-dir="${REPORT_DIR}")