From 35105fc2f0ebd50c7d6ae27f3ed7fac61665e9f3 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Thu, 17 Sep 2026 14:18:28 -0700 Subject: [PATCH 1/2] integration: drop upgrade tests from 1.7 containerd 2.4 is released, and per the documented upgrade path in RELEASES.md a direct upgrade is only supported between sequential minor releases or between sequential LTS releases. 1.7 (LTS) -> 2.3 (LTS) is the last supported jump out of 1.7, so main no longer needs to test it. Remove the 1.7 arm of TestUpgrade along with the cases that only ever ran for it: - recover-ungroupable-shim, which needed the runcv1 runtime handler and the pre-existing-config variant of the test harness. - should-address-shim-version-mismatches, which covered the task v2 bootstrap.json downgrade done for 1.7 shims. - TestIssue10467, which pinned v1.7.20 to verify the sandbox bucket migration. With those gone, oneSevenCtrdConfig, the usingExistingConfig plumbing and the runtime handler parameter are all unused, so drop them too. Signed-off-by: Maksym Pavlenko --- integration/issue10467_linux_test.go | 121 ------- integration/release_upgrade_linux_test.go | 366 +++++++--------------- 2 files changed, 121 insertions(+), 366 deletions(-) delete mode 100644 integration/issue10467_linux_test.go diff --git a/integration/issue10467_linux_test.go b/integration/issue10467_linux_test.go deleted file mode 100644 index b039635ee3..0000000000 --- a/integration/issue10467_linux_test.go +++ /dev/null @@ -1,121 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package integration - -import ( - "fmt" - "path/filepath" - "syscall" - "testing" - "time" - - "github.com/containerd/continuity/fs" - "github.com/stretchr/testify/require" - "go.etcd.io/bbolt" -) - -// TestIssue10467 tests the migration of sandboxes into the proper bucket. Prior to v1.7.21, the -// sandboxes were stored incorrectly in the root bucket. In order to verify the migration, a v1.7.20 -// must run and create a sandbox, then check the migration after upgrading to a newer version. -func TestIssue10467(t *testing.T) { - latestVersion := "v1.7.20" - - releaseBinDir := t.TempDir() - - downloadReleaseBinary(t, releaseBinDir, latestVersion) - - t.Logf("Install config for release %s", latestVersion) - workDir := t.TempDir() - oneSevenCtrdConfig(t, releaseBinDir, workDir) - - t.Log("Starting the previous release's containerd") - previousCtrdBinPath := filepath.Join(releaseBinDir, "bin", "containerd") - previousProc := newCtrdProc(t, previousCtrdBinPath, workDir, []string{"ENABLE_CRI_SANDBOXES=yes"}) - - boltdbPath := filepath.Join(workDir, "root", "io.containerd.metadata.v1.bolt", "meta.db") - - ctrdLogPath := previousProc.logPath() - t.Cleanup(func() { - if t.Failed() { - dumpFileContent(t, ctrdLogPath) - } - }) - - require.NoError(t, previousProc.isReady()) - - needToCleanup := true - t.Cleanup(func() { - if t.Failed() && needToCleanup { - t.Logf("Try to cleanup leaky pods") - cleanupPods(t, previousProc.criRuntimeService(t)) - } - }) - - t.Log("Prepare pods for current release") - upgradeCaseFuncs, hookFunc := shouldManipulateContainersInPodAfterUpgrade("")(t, 2, previousProc.criRuntimeService(t), previousProc.criImageService(t)) - upgradeCaseFunc := upgradeCaseFuncs[0] - needToCleanup = false - require.Nil(t, hookFunc) - - t.Log("Gracefully stop previous release's containerd process") - require.NoError(t, previousProc.kill(syscall.SIGTERM)) - require.NoError(t, previousProc.wait(5*time.Minute)) - - t.Logf("%s should have bucket k8s.io in root", boltdbPath) - db, err := bbolt.Open(boltdbPath, 0600, &bbolt.Options{ReadOnly: true}) - require.NoError(t, err) - require.NoError(t, db.View(func(tx *bbolt.Tx) error { - if tx.Bucket([]byte("k8s.io")) == nil { - return fmt.Errorf("expected k8s.io bucket") - } - return nil - })) - require.NoError(t, db.Close()) - - t.Log("Install default config for current release") - currentReleaseCtrdDefaultConfig(t, workDir) - - t.Log("Starting the current release's containerd") - currentProc := newCtrdProc(t, "containerd", workDir, nil) - require.NoError(t, currentProc.isReady()) - - t.Cleanup(func() { - t.Log("Cleanup all the pods") - cleanupPods(t, currentProc.criRuntimeService(t)) - - t.Log("Stopping current release's containerd process") - require.NoError(t, currentProc.kill(syscall.SIGTERM)) - require.NoError(t, currentProc.wait(5*time.Minute)) - }) - - t.Logf("%s should not have bucket k8s.io in root after restart", boltdbPath) - copiedBoltdbPath := filepath.Join(t.TempDir(), "meta.db.new") - require.NoError(t, fs.CopyFile(copiedBoltdbPath, boltdbPath)) - - db, err = bbolt.Open(copiedBoltdbPath, 0600, &bbolt.Options{ReadOnly: true}) - require.NoError(t, err) - require.NoError(t, db.View(func(tx *bbolt.Tx) error { - if tx.Bucket([]byte("k8s.io")) != nil { - return fmt.Errorf("unexpected k8s.io bucket") - } - return nil - })) - require.NoError(t, db.Close()) - - t.Log("Verifing") - upgradeCaseFunc(t, currentProc.criRuntimeService(t), currentProc.criImageService(t)) -} diff --git a/integration/release_upgrade_linux_test.go b/integration/release_upgrade_linux_test.go index 082d2139e2..22e5912458 100644 --- a/integration/release_upgrade_linux_test.go +++ b/integration/release_upgrade_linux_test.go @@ -60,52 +60,27 @@ type setupUpgradeVerifyCase func(*testing.T, int, *remote.RuntimeService, *remot // TODO: Support Windows func TestUpgrade(t *testing.T) { - for _, version := range []string{"1.7", "2.0"} { + for _, version := range []string{"2.0"} { t.Run(version, func(t *testing.T) { previousReleaseBinDir := t.TempDir() downloadPreviousLatestReleaseBinary(t, version, previousReleaseBinDir) - t.Run("recover", runUpgradeTestCase(version, previousReleaseBinDir, shouldRecoverAllThePodsAfterUpgrade)) - t.Run("exec", runUpgradeTestCase(version, previousReleaseBinDir, execToExistingContainer)) - t.Run("manipulate", runUpgradeTestCase(version, previousReleaseBinDir, shouldManipulateContainersInPodAfterUpgrade("" /* default runtime */))) + t.Run("recover", runUpgradeTestCase(previousReleaseBinDir, shouldRecoverAllThePodsAfterUpgrade)) + t.Run("exec", runUpgradeTestCase(previousReleaseBinDir, execToExistingContainer)) + t.Run("manipulate", runUpgradeTestCase(previousReleaseBinDir, shouldManipulateContainersInPodAfterUpgrade)) - t.Run("recover-images", runUpgradeTestCase(version, previousReleaseBinDir, shouldRecoverExistingImages)) - t.Run("metrics", runUpgradeTestCase(version, previousReleaseBinDir, shouldParseMetricDataCorrectly)) + t.Run("recover-images", runUpgradeTestCase(previousReleaseBinDir, shouldRecoverExistingImages)) + t.Run("metrics", runUpgradeTestCase(previousReleaseBinDir, shouldParseMetricDataCorrectly)) t.Run("kill-shim-before-delete-task", - runUpgradeTestCase(version, previousReleaseBinDir, + runUpgradeTestCase(previousReleaseBinDir, shouldHandleShimExitStatusAfterUpgrade(previousReleaseBinDir))) - - if version == "1.7" { - t.Run("recover-ungroupable-shim", runUpgradeTestCaseWithExistingConfig(version, - previousReleaseBinDir, true, shouldManipulateContainersInPodAfterUpgrade("runcv1"))) - - t.Run("should-address-shim-version-mismatches", - runUpgradeTestCase(version, previousReleaseBinDir, shouldAdjustShimVersionDuringRestarting)) - } }) } } func runUpgradeTestCase( - previousVersion string, previousReleaseBinDir string, setupUpgradeVerifyCase func(*testing.T, int, *remote.RuntimeService, *remote.ImageService) ([]upgradeVerifyCaseFunc, beforeUpgradeHookFunc), -) func(t *testing.T) { - return runUpgradeTestCaseWithExistingConfig( - previousVersion, - previousReleaseBinDir, - // use new empty configuration so that we could use new shim - // binary to cleanup resources created by old shim binary - false, - setupUpgradeVerifyCase, - ) -} - -func runUpgradeTestCaseWithExistingConfig( - previousVersion string, - previousReleaseBinDir string, - usingExistingConfig bool, - setupUpgradeVerifyCase func(*testing.T, int, *remote.RuntimeService, *remote.ImageService) ([]upgradeVerifyCaseFunc, beforeUpgradeHookFunc), ) func(t *testing.T) { return func(t *testing.T) { // NOTE: Using t.TempDir() here is to ensure there are no leaky @@ -113,14 +88,8 @@ func runUpgradeTestCaseWithExistingConfig( workDir := t.TempDir() t.Log("Install config for previous release") - var taskVersion int - if previousVersion == "1.7" { - oneSevenCtrdConfig(t, previousReleaseBinDir, workDir) - taskVersion = 2 - } else { - previousReleaseCtrdConfig(t, previousReleaseBinDir, workDir) - taskVersion = 3 - } + previousReleaseCtrdConfig(t, previousReleaseBinDir, workDir) + const taskVersion = 3 t.Log("Starting the previous release's containerd") previousCtrdBinPath := filepath.Join(previousReleaseBinDir, "bin", "containerd") @@ -154,10 +123,10 @@ func runUpgradeTestCaseWithExistingConfig( hookFunc(t) } - if !usingExistingConfig { - t.Log("Install default config for current release") - currentReleaseCtrdDefaultConfig(t, workDir) - } + // Use new empty configuration so that we could use new shim + // binary to cleanup resources created by old shim binary. + t.Log("Install default config for current release") + currentReleaseCtrdDefaultConfig(t, workDir) t.Log("Starting the current release's containerd") currentProc := newCtrdProc(t, "containerd", workDir, nil) @@ -188,67 +157,6 @@ func runUpgradeTestCaseWithExistingConfig( } } -// shouldAdjustShimVersionDuringRestarting verifies that the shim manager -// can handle shim proto version mismatches during a containerd restart. -// -// Steps: -// 1. Use containerd-shim-runc-v2 from v1.7.x to set up a running pod. -// 2. After upgrading, use the new containerd-shim-runc-v2 to create a new container in the same pod. -// The new shim returns bootstrap.json with version 3. -// The shim manager auto-downgrades the version to 2, but does not update bootstrap.json. -// 3. Restart the containerd process; the new container should be recovered successfully. -func shouldAdjustShimVersionDuringRestarting(t *testing.T, _ int, - rSvc *remote.RuntimeService, iSvc *remote.ImageService) ([]upgradeVerifyCaseFunc, beforeUpgradeHookFunc) { - - var busyboxImage = images.Get(images.BusyBox) - - pullImagesByCRI(t, iSvc, busyboxImage) - - podCtx := newPodTCtx(t, rSvc, "running-pod", "sandbox") - - cntr1 := podCtx.createContainer("running", busyboxImage, - criruntime.ContainerState_CONTAINER_RUNNING, - WithCommand("sleep", "1d")) - - var cntr2 string - - createNewContainerInPodFunc := func(t *testing.T, rSvc *remote.RuntimeService, _ *remote.ImageService) { - t.Log("Creating new container in the previous pod") - cntr2 = podCtx.createContainer("new-container", busyboxImage, - criruntime.ContainerState_CONTAINER_RUNNING, - WithCommand("sleep", "1d")) - } - - shouldBeRunningFunc := func(t *testing.T, rSvc *remote.RuntimeService, _ *remote.ImageService) { - t.Log("Checking the running container in the previous pod") - - pods, err := rSvc.ListPodSandbox(nil) - require.NoError(t, err) - require.Len(t, pods, 1) - - cntrs, err := rSvc.ListContainers(&criruntime.ContainerFilter{ - PodSandboxId: pods[0].Id, - }) - require.NoError(t, err) - require.Len(t, cntrs, 2) - - for _, cntr := range cntrs { - switch cntr.Id { - case cntr1: - assert.Equal(t, criruntime.ContainerState_CONTAINER_RUNNING.String(), cntr.State.String()) - case cntr2: - assert.Equal(t, criruntime.ContainerState_CONTAINER_RUNNING.String(), cntr.State.String()) - default: - t.Errorf("unexpected container %s in pod %s", cntr.Id, pods[0].Id) - } - } - } - return []upgradeVerifyCaseFunc{ - createNewContainerInPodFunc, - shouldBeRunningFunc, - }, nil -} - func shouldRecoverAllThePodsAfterUpgrade(t *testing.T, taskVersion int, rSvc *remote.RuntimeService, iSvc *remote.ImageService) ([]upgradeVerifyCaseFunc, beforeUpgradeHookFunc) { @@ -405,128 +313,127 @@ func getFileSize(t *testing.T, filePath string) int64 { return st.Size() } -func shouldManipulateContainersInPodAfterUpgrade(runtimeHandler string) setupUpgradeVerifyCase { - return func(t *testing.T, taskVersion int, rSvc *remote.RuntimeService, iSvc *remote.ImageService) ([]upgradeVerifyCaseFunc, beforeUpgradeHookFunc) { - shimConns := []shimConn{} +func shouldManipulateContainersInPodAfterUpgrade(t *testing.T, _ int, + rSvc *remote.RuntimeService, iSvc *remote.ImageService) ([]upgradeVerifyCaseFunc, beforeUpgradeHookFunc) { + shimConns := []shimConn{} - var busyboxImage = images.Get(images.BusyBox) + var busyboxImage = images.Get(images.BusyBox) - pullImagesByCRI(t, iSvc, busyboxImage) + pullImagesByCRI(t, iSvc, busyboxImage) - podCtx := newPodTCtxWithRuntimeHandler(t, rSvc, "running-pod", "sandbox", runtimeHandler) + podCtx := newPodTCtx(t, rSvc, "running-pod", "sandbox") - cntr1 := podCtx.createContainer("running", busyboxImage, + cntr1 := podCtx.createContainer("running", busyboxImage, + criruntime.ContainerState_CONTAINER_RUNNING, + WithCommand("sleep", "1d")) + + t.Logf("Building shim connect for container %s", cntr1) + shimConns = append(shimConns, buildShimClientFromBundle(t, rSvc, cntr1)) + + cntr2 := podCtx.createContainer("created", busyboxImage, + criruntime.ContainerState_CONTAINER_CREATED, + WithCommand("sleep", "1d")) + + cntr3 := podCtx.createContainer("stopped", busyboxImage, + criruntime.ContainerState_CONTAINER_EXITED, + WithCommand("sleep", "1d")) + + verifyFunc := func(t *testing.T, rSvc *remote.RuntimeService, _ *remote.ImageService) { + // TODO(fuweid): make svc re-connect to new socket + podCtx.rSvc = rSvc + + t.Log("Manipulating containers in the previous pod") + + // For the running container, we get status and stats of it, + // exec and execsync in it, stop and remove it + checkContainerState(t, rSvc, cntr1, criruntime.ContainerState_CONTAINER_RUNNING) + + t.Logf("Preparing attachable exec for container %s", cntr1) + _, err := rSvc.Exec(&criruntime.ExecRequest{ + ContainerId: cntr1, + Cmd: []string{"/bin/sh"}, + Stderr: false, + Stdout: true, + Stdin: true, + Tty: true, + }) + require.NoError(t, err) + + t.Logf("Stopping container %s", cntr1) + require.NoError(t, rSvc.StopContainer(cntr1, 0)) + checkContainerState(t, rSvc, cntr1, criruntime.ContainerState_CONTAINER_EXITED) + + cntr1DataDir := podCtx.containerDataDir(cntr1) + t.Logf("Container %s's data dir %s should be remained until RemoveContainer", cntr1, cntr1DataDir) + _, err = os.Stat(cntr1DataDir) + require.NoError(t, err) + + t.Logf("Starting created container %s", cntr2) + checkContainerState(t, rSvc, cntr2, criruntime.ContainerState_CONTAINER_CREATED) + + require.NoError(t, rSvc.StartContainer(cntr2)) + checkContainerState(t, rSvc, cntr2, criruntime.ContainerState_CONTAINER_RUNNING) + + t.Logf("Building shim connect for container %s", cntr2) + shimConns = append(shimConns, buildShimClientFromBundle(t, rSvc, cntr2)) + + t.Logf("Stopping running container %s", cntr2) + require.NoError(t, rSvc.StopContainer(cntr2, 0)) + checkContainerState(t, rSvc, cntr2, criruntime.ContainerState_CONTAINER_EXITED) + + t.Logf("Removing exited container %s", cntr3) + checkContainerState(t, rSvc, cntr3, criruntime.ContainerState_CONTAINER_EXITED) + + cntr3DataDir := podCtx.containerDataDir(cntr3) + _, err = os.Stat(cntr3DataDir) + require.NoError(t, err) + + require.NoError(t, rSvc.RemoveContainer(cntr3)) + + t.Logf("Container %s's data dir %s should be deleted after RemoveContainer", cntr3, cntr3DataDir) + _, err = os.Stat(cntr3DataDir) + require.True(t, os.IsNotExist(err)) + + // Create a new container in the previous pod, start, stop, and remove it + cntr4 := podCtx.createContainer("runinpreviouspod", busyboxImage, criruntime.ContainerState_CONTAINER_RUNNING, WithCommand("sleep", "1d")) - t.Logf("Building shim connect for container %s", cntr1) - shimConns = append(shimConns, buildShimClientFromBundle(t, rSvc, cntr1)) + t.Logf("Building shim connect for container %s", cntr4) + shimConns = append(shimConns, buildShimClientFromBundle(t, rSvc, cntr4)) - cntr2 := podCtx.createContainer("created", busyboxImage, - criruntime.ContainerState_CONTAINER_CREATED, + podCtx.stop(true) + podDataDir := podCtx.dataDir() + + t.Logf("Pod %s's data dir %s should be deleted", podCtx.id, podDataDir) + _, err = os.Stat(podDataDir) + require.True(t, os.IsNotExist(err)) + + cntrDataDir := filepath.Dir(cntr3DataDir) + t.Logf("Containers data dir %s should be empty", cntrDataDir) + ents, err := os.ReadDir(cntrDataDir) + require.NoError(t, err) + require.Len(t, ents, 0, cntrDataDir) + + t.Log("Creating new running container in new pod") + pod2Ctx := newPodTCtx(t, rSvc, "running-pod-2", "sandbox") + pod2Cntr := pod2Ctx.createContainer("running", busyboxImage, + criruntime.ContainerState_CONTAINER_RUNNING, WithCommand("sleep", "1d")) - cntr3 := podCtx.createContainer("stopped", busyboxImage, - criruntime.ContainerState_CONTAINER_EXITED, - WithCommand("sleep", "1d")) + t.Logf("Building shim connect for container %s", pod2Cntr) + shimConns = append(shimConns, buildShimClientFromBundle(t, rSvc, pod2Cntr)) - verifyFunc := func(t *testing.T, rSvc *remote.RuntimeService, _ *remote.ImageService) { - // TODO(fuweid): make svc re-connect to new socket - podCtx.rSvc = rSvc + pod2Ctx.stop(true) - t.Log("Manipulating containers in the previous pod") - - // For the running container, we get status and stats of it, - // exec and execsync in it, stop and remove it - checkContainerState(t, rSvc, cntr1, criruntime.ContainerState_CONTAINER_RUNNING) - - t.Logf("Preparing attachable exec for container %s", cntr1) - _, err := rSvc.Exec(&criruntime.ExecRequest{ - ContainerId: cntr1, - Cmd: []string{"/bin/sh"}, - Stderr: false, - Stdout: true, - Stdin: true, - Tty: true, - }) - require.NoError(t, err) - - t.Logf("Stopping container %s", cntr1) - require.NoError(t, rSvc.StopContainer(cntr1, 0)) - checkContainerState(t, rSvc, cntr1, criruntime.ContainerState_CONTAINER_EXITED) - - cntr1DataDir := podCtx.containerDataDir(cntr1) - t.Logf("Container %s's data dir %s should be remained until RemoveContainer", cntr1, cntr1DataDir) - _, err = os.Stat(cntr1DataDir) - require.NoError(t, err) - - t.Logf("Starting created container %s", cntr2) - checkContainerState(t, rSvc, cntr2, criruntime.ContainerState_CONTAINER_CREATED) - - require.NoError(t, rSvc.StartContainer(cntr2)) - checkContainerState(t, rSvc, cntr2, criruntime.ContainerState_CONTAINER_RUNNING) - - t.Logf("Building shim connect for container %s", cntr2) - shimConns = append(shimConns, buildShimClientFromBundle(t, rSvc, cntr2)) - - t.Logf("Stopping running container %s", cntr2) - require.NoError(t, rSvc.StopContainer(cntr2, 0)) - checkContainerState(t, rSvc, cntr2, criruntime.ContainerState_CONTAINER_EXITED) - - t.Logf("Removing exited container %s", cntr3) - checkContainerState(t, rSvc, cntr3, criruntime.ContainerState_CONTAINER_EXITED) - - cntr3DataDir := podCtx.containerDataDir(cntr3) - _, err = os.Stat(cntr3DataDir) - require.NoError(t, err) - - require.NoError(t, rSvc.RemoveContainer(cntr3)) - - t.Logf("Container %s's data dir %s should be deleted after RemoveContainer", cntr3, cntr3DataDir) - _, err = os.Stat(cntr3DataDir) - require.True(t, os.IsNotExist(err)) - - // Create a new container in the previous pod, start, stop, and remove it - cntr4 := podCtx.createContainer("runinpreviouspod", busyboxImage, - criruntime.ContainerState_CONTAINER_RUNNING, - WithCommand("sleep", "1d")) - - t.Logf("Building shim connect for container %s", cntr4) - shimConns = append(shimConns, buildShimClientFromBundle(t, rSvc, cntr4)) - - podCtx.stop(true) - podDataDir := podCtx.dataDir() - - t.Logf("Pod %s's data dir %s should be deleted", podCtx.id, podDataDir) - _, err = os.Stat(podDataDir) - require.True(t, os.IsNotExist(err)) - - cntrDataDir := filepath.Dir(cntr3DataDir) - t.Logf("Containers data dir %s should be empty", cntrDataDir) - ents, err := os.ReadDir(cntrDataDir) - require.NoError(t, err) - require.Len(t, ents, 0, cntrDataDir) - - t.Log("Creating new running container in new pod") - pod2Ctx := newPodTCtxWithRuntimeHandler(t, rSvc, "running-pod-2", "sandbox", runtimeHandler) - pod2Cntr := pod2Ctx.createContainer("running", busyboxImage, - criruntime.ContainerState_CONTAINER_RUNNING, - WithCommand("sleep", "1d")) - - t.Logf("Building shim connect for container %s", pod2Cntr) - shimConns = append(shimConns, buildShimClientFromBundle(t, rSvc, pod2Cntr)) - - pod2Ctx.stop(true) - - // If connection is closed, it means the shim process exits. - for _, shimCli := range shimConns { - t.Logf("Checking container %s's shim client", shimCli.cntrID) - _, err = shimCli.cli.Connect(context.Background(), &apitask.ConnectRequest{}) - assert.ErrorContains(t, err, "ttrpc: closed", "should be closed after deleting pod") - } + // If connection is closed, it means the shim process exits. + for _, shimCli := range shimConns { + t.Logf("Checking container %s's shim client", shimCli.cntrID) + _, err = shimCli.cli.Connect(context.Background(), &apitask.ConnectRequest{}) + assert.ErrorContains(t, err, "ttrpc: closed", "should be closed after deleting pod") } - return []upgradeVerifyCaseFunc{verifyFunc}, nil } + return []upgradeVerifyCaseFunc{verifyFunc}, nil } func shouldRecoverExistingImages(t *testing.T, _ int, @@ -629,14 +536,8 @@ done func newPodTCtx(t *testing.T, rSvc *remote.RuntimeService, name, ns string, opts ...PodSandboxOpts) *podTCtx { - return newPodTCtxWithRuntimeHandler(t, rSvc, name, ns, "", opts...) -} - -func newPodTCtxWithRuntimeHandler(t *testing.T, rSvc *remote.RuntimeService, - name, ns, runtimeHandler string, opts ...PodSandboxOpts) *podTCtx { - sbConfig := PodSandboxConfig(name, ns, opts...) - sbID, err := rSvc.RunPodSandbox(sbConfig, runtimeHandler) + sbID, err := rSvc.RunPodSandbox(sbConfig, "" /* default runtime */) require.NoError(t, err) return &podTCtx{ @@ -872,31 +773,6 @@ version = 2 require.NoError(t, err, "failed to create config for previous release") } -// previousReleaseCtrdConfig generates containerd config with previous release -// shim binary. -func oneSevenCtrdConfig(t *testing.T, previousReleaseBinDir, targetDir string) { - // TODO(fuweid): - // - // We should choose correct config version based on previous release. - // Currently, we're focusing on v1.x -> v2.0 so we use version = 2 here. - rawCfg := fmt.Sprintf(` -version = 2 - -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - runtime_path = "%s/bin/containerd-shim-runc-v2" - - -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runcv1] - runtime_type = "io.containerd.runc.v2" - runtime_path = "%s/bin/containerd-shim-runc-v1" -`, previousReleaseBinDir, previousReleaseBinDir) - - fileName := filepath.Join(targetDir, "config.toml") - err := os.WriteFile(fileName, []byte(rawCfg), 0600) - require.NoError(t, err, "failed to create config for previous release") -} - // criRuntimeService returns a CRI runtime client based on the grpc address. func (p *ctrdProc) criRuntimeService(t *testing.T) *remote.RuntimeService { service, err := remote.NewRuntimeService(p.grpcAddress(), 1*time.Minute) From e4e83b3af8febab7b340663eafad9acde4749679 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Thu, 17 Sep 2026 14:19:00 -0700 Subject: [PATCH 2/2] integration: test upgrades from 2.3 and 2.4 With 1.7 gone the matrix was left testing only 2.0, which is neither the current LTS nor the release immediately preceding main. RELEASES.md promises two paths into a given release: a sequential minor upgrade and a sequential LTS upgrade. For main those are 2.4 and 2.3 respectively, so test both. Switch the download to the static release tarball. Release binaries are built on Ubuntu 22.04, and the dynamically linked containerd from 2.3 on needs GLIBC_2.32 and GLIBC_2.34. The EL8 hosts kept in the VM integration matrix for their old glibc only have 2.28, so the previous release never started there and every case timed out waiting for it. The 2.0 binaries needed nothing newer than GLIBC_2.4, which is why the old matrix did not hit this. The static build has no libc dependency, and its shim is the same binary the dynamic tarball already shipped. Also bump the previous-release config to version 3. Version 2 only mattered while 1.x was in the matrix, and every release now under test writes version 3 by default. kill-shim-before-delete-task needs its expected exit status picked per version. Once the shim has been killed, containerd reports whatever ` delete` returns, and manager.Stop prefers the exit status recorded in the bundle, falling back to a synthesized SIGKILL status when there is no record. Recording landed in v2.4.0, so a bundle left by 2.3 reports 128+SIGKILL while one left by 2.4 reports the status the task actually exited with. The old matrix predated recording entirely and only ever covered the fallback; keying the expectation off the previous version covers both branches. Signed-off-by: Maksym Pavlenko --- .../release_upgrade_exit_status_linux_test.go | 17 ++++++++++++++--- integration/release_upgrade_linux_test.go | 14 +++++++------- integration/release_upgrade_utils_linux_test.go | 7 ++++++- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/integration/release_upgrade_exit_status_linux_test.go b/integration/release_upgrade_exit_status_linux_test.go index e86f0dcf7a..3f41c36609 100644 --- a/integration/release_upgrade_exit_status_linux_test.go +++ b/integration/release_upgrade_exit_status_linux_test.go @@ -29,6 +29,7 @@ import ( "github.com/containerd/errdefs" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" + "golang.org/x/mod/semver" "google.golang.org/protobuf/testing/protocmp" eventtypes "github.com/containerd/containerd/api/events" @@ -41,12 +42,22 @@ import ( "github.com/containerd/containerd/v2/plugins" ) -func shouldHandleShimExitStatusAfterUpgrade(previousReleaseBinDir string) setupUpgradeVerifyCase { +func shouldHandleShimExitStatusAfterUpgrade(previousReleaseBinDir, previousVersion string) setupUpgradeVerifyCase { const ( shimExitStatus uint32 = 42 shimExitStatusNamespace = "kill-shim-before-delete-task" ) + // Shims from v2.4.0 on record the task's exit status in the bundle, so + // cleaning up after the killed shim reports the status the task actually + // exited with. Bundles left by older shims carry no such record and the + // cleanup falls back to a synthesized SIGKILL status. Both branches live in + // manager.Stop in cmd/containerd-shim-runc-v2/manager. + killedShimExitStatus := uint32(128 + syscall.SIGKILL) + if semver.Compare("v"+previousVersion, "v2.4") >= 0 { + killedShimExitStatus = shimExitStatus + } + return func(t *testing.T, _ int, rSvc *remote.RuntimeService, _ *remote.ImageService) ([]upgradeVerifyCaseFunc, beforeUpgradeHookFunc) { client := newUpgradeContainerdClient(t, rSvc, shimExitStatusNamespace) ctx := t.Context() @@ -130,12 +141,12 @@ func shouldHandleShimExitStatusAfterUpgrade(previousReleaseBinDir string) setupU ContainerID: id, ID: id, Pid: taskPID, - ExitStatus: uint32(128 + syscall.SIGKILL), + ExitStatus: killedShimExitStatus, }}, {Delete: &eventtypes.TaskDelete{ ContainerID: id, Pid: taskPID, - ExitStatus: uint32(128 + syscall.SIGKILL), + ExitStatus: killedShimExitStatus, }}, } if diff := cmp.Diff(wantEvents, gotEvents, cmpOpts...); diff != "" { diff --git a/integration/release_upgrade_linux_test.go b/integration/release_upgrade_linux_test.go index 22e5912458..390772189f 100644 --- a/integration/release_upgrade_linux_test.go +++ b/integration/release_upgrade_linux_test.go @@ -60,7 +60,7 @@ type setupUpgradeVerifyCase func(*testing.T, int, *remote.RuntimeService, *remot // TODO: Support Windows func TestUpgrade(t *testing.T) { - for _, version := range []string{"2.0"} { + for _, version := range []string{"2.3", "2.4"} { t.Run(version, func(t *testing.T) { previousReleaseBinDir := t.TempDir() downloadPreviousLatestReleaseBinary(t, version, previousReleaseBinDir) @@ -73,7 +73,7 @@ func TestUpgrade(t *testing.T) { t.Run("kill-shim-before-delete-task", runUpgradeTestCase(previousReleaseBinDir, - shouldHandleShimExitStatusAfterUpgrade(previousReleaseBinDir))) + shouldHandleShimExitStatusAfterUpgrade(previousReleaseBinDir, version))) }) } } @@ -755,12 +755,12 @@ func currentReleaseCtrdDefaultConfig(t *testing.T, targetDir string) { // previousReleaseCtrdConfig generates containerd config with previous release // shim binary. func previousReleaseCtrdConfig(t *testing.T, previousReleaseBinDir, targetDir string) { - // TODO(fuweid): - // - // We should choose correct config version based on previous release. - // Currently, we're focusing on v1.x -> v2.0 so we use version = 2 here. + // NOTE: Config version 3 is the oldest version understood by every + // release in the test matrix (see the configuration version table in + // RELEASES.md). Bump it once the matrix no longer includes a release + // predating the next version. rawCfg := fmt.Sprintf(` -version = 2 +version = 3 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] runtime_type = "io.containerd.runc.v2" diff --git a/integration/release_upgrade_utils_linux_test.go b/integration/release_upgrade_utils_linux_test.go index 530e32b076..f715f2fb6f 100644 --- a/integration/release_upgrade_utils_linux_test.go +++ b/integration/release_upgrade_utils_linux_test.go @@ -41,8 +41,13 @@ func downloadPreviousLatestReleaseBinary(t *testing.T, version, targetDir string } // downloadReleaseBinary downloads containerd binary with a given release. +// +// NOTE: Use the static build. Release binaries are built on Ubuntu 22.04, so +// the dynamically linked containerd needs GLIBC_2.32 and GLIBC_2.34, which the +// EL8 hosts kept in the VM integration matrix for their old glibc cannot +// provide. The static build has no libc dependency and runs everywhere. func downloadReleaseBinary(t *testing.T, targetDir string, ver string) { - targetURL := fmt.Sprintf("https://github.com/containerd/containerd/releases/download/%s/containerd-%s-linux-%s.tar.gz", + targetURL := fmt.Sprintf("https://github.com/containerd/containerd/releases/download/%s/containerd-static-%s-linux-%s.tar.gz", ver, strings.TrimPrefix(ver, "v"), runtime.GOARCH, )