containerd/integration/sandbox_run_rollback_test.go

//go:build linux

/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package integration

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"syscall"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	criapiv1 "k8s.io/cri-api/pkg/apis/runtime/v1"

	podsandboxtypes "github.com/containerd/containerd/v2/internal/cri/server/podsandbox/types"
	"github.com/containerd/containerd/v2/internal/failpoint"
)

const (
	failpointRuntimeHandler = "runc-fp"
	failpointCNIBinary      = "cni-bridge-fp"

	failpointShimPrefixKey = "io.containerd.runtime.v2.shim.failpoint."

	failpointCNIConfPathKey = "failpoint.cni.containerd.io/confpath"
)

func TestRunPodSandboxWithSetupCNIFailure(t *testing.T) {
	if runtime.GOOS != "linux" {
		t.Skip()
	}

	t.Logf("Inject CNI failpoint")
	conf := &failpointConf{
		Add: "1*error(you-shall-not-pass!)",
	}

	sbConfig := PodSandboxConfig(t.Name(), "failpoint")
	injectCNIFailpoint(t, sbConfig, conf)

	t.Logf("Create a sandbox")
	_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
	require.Error(t, err)
	require.ErrorContains(t, err, "you-shall-not-pass!")

	t.Logf("Retry to create sandbox with same config")
	sb, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
	require.NoError(t, err)

	err = runtimeService.StopPodSandbox(sb)
	require.NoError(t, err)

	err = runtimeService.RemovePodSandbox(sb)
	require.NoError(t, err)
}

func TestRunPodSandboxWithShimStartFailure(t *testing.T) {
	if runtime.GOOS != "linux" {
		t.Skip()
	}

	t.Logf("Inject Shim failpoint")

	sbConfig := PodSandboxConfig(t.Name(), "failpoint")
	injectShimFailpoint(t, sbConfig, map[string]string{
		"Start": "1*error(no hard feelings)",
	})

	t.Logf("Create a sandbox")
	_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
	require.Error(t, err)
	require.ErrorContains(t, err, "no hard feelings")
}

// TestRunPodSandboxWithShimDeleteFailure should keep the sandbox record if
// failed to rollback shim by shim.Delete API.
func TestRunPodSandboxWithShimDeleteFailure(t *testing.T) {
	if runtime.GOOS != "linux" {
		t.Skip()
	}

	testCase := func(restart bool) func(*testing.T) {
		return func(t *testing.T) {
			t.Log("Init PodSandboxConfig with specific label")
			labels := map[string]string{
				t.Name(): "true",
			}
			sbConfig := PodSandboxConfig(t.Name(), "failpoint", WithPodLabels(labels))

			t.Log("Inject Shim failpoint")
			injectShimFailpoint(t, sbConfig, map[string]string{
				"Start": "1*error(failed to start shim)",
				// Kill invoked by Delete during rollback, we should block it instead of Delete
				//
				// NOTE: If we allow to Kill, there could be a race condition. The event handler
				// will cleanup the sandbox record after Kill asynchronously. It could cause that
				// ListPodSandbox returns empty.
				"Kill": "1*error(please retry)",
			})

			t.Log("Create a sandbox")
			_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
			require.Error(t, err)
			require.ErrorContains(t, err, "failed to start shim")

			t.Log("ListPodSandbox with the specific label")
			l, err := runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{LabelSelector: labels})
			require.NoError(t, err)
			require.Len(t, l, 1)

			sb := l[0]
			require.Equal(t, criapiv1.PodSandboxState_SANDBOX_NOTREADY, sb.State)
			require.Equal(t, sbConfig.Metadata.Name, sb.Metadata.Name)
			require.Equal(t, sbConfig.Metadata.Namespace, sb.Metadata.Namespace)
			require.Equal(t, sbConfig.Metadata.Uid, sb.Metadata.Uid)
			require.Equal(t, sbConfig.Metadata.Attempt, sb.Metadata.Attempt)

			t.Log("Check PodSandboxStatus")
			sbStatus, err := runtimeService.PodSandboxStatus(sb.Id)
			require.NoError(t, err)
			require.Equal(t, criapiv1.PodSandboxState_SANDBOX_NOTREADY, sbStatus.State)
			require.Greater(t, len(sbStatus.Network.Ip), 0)

			if restart {
				t.Log("Restart containerd")
				RestartContainerd(t, syscall.SIGTERM)

				t.Log("ListPodSandbox with the specific label")
				l, err = runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{Id: sb.Id})
				require.NoError(t, err)
				require.Len(t, l, 1)
				require.Equal(t, criapiv1.PodSandboxState_SANDBOX_NOTREADY, l[0].State)

				t.Log("Check PodSandboxStatus")
				sbStatus, err := runtimeService.PodSandboxStatus(sb.Id)
				require.NoError(t, err)
				t.Log(sbStatus.Network)
				require.Equal(t, criapiv1.PodSandboxState_SANDBOX_NOTREADY, sbStatus.State)
			}

			t.Log("Cleanup leaky sandbox")
			err = runtimeService.RemovePodSandbox(sb.Id)
			require.NoError(t, err)
		}
	}

	t.Run("CleanupAfterRestart", testCase(true))
	t.Run("JustCleanup", testCase(false))
}

// TestRunPodSandboxWithShimStartAndTeardownCNIFailure should keep the sandbox
// record if failed to rollback CNI API.
func TestRunPodSandboxWithShimStartAndTeardownCNIFailure(t *testing.T) {
	if runtime.GOOS != "linux" {
		t.Skip()
	}

	testCase := func(restart bool) func(*testing.T) {
		return func(t *testing.T) {
			t.Log("Init PodSandboxConfig with specific key")
			labels := map[string]string{
				t.Name(): "true",
			}
			sbConfig := PodSandboxConfig(t.Name(), "failpoint", WithPodLabels(labels))

			t.Log("Inject Shim failpoint")
			injectShimFailpoint(t, sbConfig, map[string]string{
				"Start": "1*error(failed to start shim)",
			})

			t.Log("Inject CNI failpoint")
			conf := &failpointConf{
				Del: "1*error(please retry)",
			}
			injectCNIFailpoint(t, sbConfig, conf)

			t.Log("Create a sandbox")
			_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
			require.Error(t, err)
			require.ErrorContains(t, err, "failed to start shim")

			t.Log("ListPodSandbox with the specific label")
			l, err := runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{LabelSelector: labels})
			require.NoError(t, err)
			require.Len(t, l, 1)

			sb := l[0]
			require.Equal(t, criapiv1.PodSandboxState_SANDBOX_NOTREADY, sb.State)
			require.Equal(t, sbConfig.Metadata.Name, sb.Metadata.Name)
			require.Equal(t, sbConfig.Metadata.Namespace, sb.Metadata.Namespace)
			require.Equal(t, sbConfig.Metadata.Uid, sb.Metadata.Uid)
			require.Equal(t, sbConfig.Metadata.Attempt, sb.Metadata.Attempt)

			if restart {
				t.Log("Restart containerd")
				RestartContainerd(t, syscall.SIGTERM)

				t.Log("ListPodSandbox with the specific label")
				l, err = runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{Id: sb.Id})
				require.NoError(t, err)
				require.Len(t, l, 1)
				require.Equal(t, criapiv1.PodSandboxState_SANDBOX_NOTREADY, l[0].State)
			}

			t.Log("Cleanup leaky sandbox")
			err = runtimeService.RemovePodSandbox(sb.Id)
			require.NoError(t, err)
		}
	}
	t.Run("CleanupAfterRestart", testCase(true))
	t.Run("JustCleanup", testCase(false))
}

// TestRunPodSandboxWithShimStartAndTeardownCNISlow should keep the sandbox
// record if failed to rollback CNI API.
func TestRunPodSandboxAndTeardownCNISlow(t *testing.T) {
	if runtime.GOOS != "linux" {
		t.Skip()
	}

	t.Log("Init PodSandboxConfig with specific key")
	sbName := t.Name()
	labels := map[string]string{
		sbName: "true",
	}
	sbConfig := PodSandboxConfig(sbName, "failpoint", WithPodLabels(labels))

	t.Log("Inject CNI failpoint")
	conf := &failpointConf{
		// Delay 1 day
		Add: "1*delay(86400000)",
	}
	injectCNIFailpoint(t, sbConfig, conf)

	var wg sync.WaitGroup

	wg.Go(func() {
		t.Log("Create a sandbox")
		_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
		require.Error(t, err)
		require.ErrorContains(t, err, "error reading from server: EOF")
	})

	assert.NoError(t, ensureCNIAddRunning(t, sbName), "check that failpoint CNI.Add is running")

	// Use SIGKILL to prevent containerd server gracefulshutdown which may cause indeterministic invocation of defer functions
	t.Log("Restart containerd")
	RestartContainerd(t, syscall.SIGKILL)

	wg.Wait()

	t.Log("ListPodSandbox with the specific label")
	l, err := runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{LabelSelector: labels})
	require.NoError(t, err)
	require.Len(t, l, 1)

	sb := l[0]

	defer func() {
		t.Log("Cleanup leaky sandbox")
		err := runtimeService.StopPodSandbox(sb.Id)
		assert.NoError(t, err)
		err = runtimeService.RemovePodSandbox(sb.Id)
		require.NoError(t, err)
	}()

	assert.Equal(t, criapiv1.PodSandboxState_SANDBOX_NOTREADY, sb.State)
	assert.Equal(t, sbConfig.Metadata.Name, sb.Metadata.Name)
	assert.Equal(t, sbConfig.Metadata.Namespace, sb.Metadata.Namespace)
	assert.Equal(t, sbConfig.Metadata.Uid, sb.Metadata.Uid)
	assert.Equal(t, sbConfig.Metadata.Attempt, sb.Metadata.Attempt)

	t.Log("Get sandbox info (sbserver)")
	_, info, err := sbserverSandboxInfo(sb.Id)
	require.NoError(t, err)
	require.False(t, info.NetNSClosed)

	assert.NotEmpty(t, info.Metadata.NetNSPath, "network namespace should be set")
}

// sbserverSandboxInfo gets sandbox info.
func sbserverSandboxInfo(id string) (*criapiv1.PodSandboxStatus, *podsandboxtypes.SandboxInfo, error) {
	client, conn, err := RawRuntimeClient()
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get raw runtime client: %w", err)
	}
	defer conn.Close()
	resp, err := client.PodSandboxStatus(context.Background(), &criapiv1.PodSandboxStatusRequest{
		PodSandboxId: id,
		Verbose:      true,
	})
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get sandbox status: %w", err)
	}
	status := resp.GetStatus()
	var info podsandboxtypes.SandboxInfo
	if err := json.Unmarshal([]byte(resp.GetInfo()["info"]), &info); err != nil {
		return nil, nil, fmt.Errorf("failed to unmarshal sandbox info: %w", err)
	}
	return status, &info, nil
}

func ensureCNIAddRunning(t *testing.T, sbName string) error {
	return Eventually(func() (bool, error) {
		pids, err := PidsOf(failpointCNIBinary)
		if err != nil || len(pids) == 0 {
			return false, err
		}

		for _, pid := range pids {
			envs, err := PidEnvs(pid)
			if err != nil {
				t.Logf("failed to read environ of pid %v: %v: skip it", pid, err)
				continue
			}

			args, ok := envs["CNI_ARGS"]
			if !ok {
				t.Logf("expected CNI_ARGS env but got nothing, skip pid=%v", pid)
				continue
			}

			for arg := range strings.SplitSeq(args, ";") {
				if arg == "K8S_POD_NAME="+sbName {
					return true, nil
				}
			}
		}
		return false, nil
	}, time.Second, 30*time.Second)
}

// failpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint.
type failpointConf struct {
	Add   string `json:"cmdAdd"`
	Del   string `json:"cmdDel"`
	Check string `json:"cmdCheck"`
}

func injectCNIFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, conf *failpointConf) {
	stateDir := t.TempDir()

	metadata := sbConfig.Metadata
	fpFilename := filepath.Join(stateDir,
		fmt.Sprintf("%s-%s.json", metadata.Namespace, strings.Replace(metadata.Name, "/", "-", -1)))

	data, err := json.Marshal(conf)
	require.NoError(t, err)

	err = os.WriteFile(fpFilename, data, 0666)
	require.NoError(t, err)

	sbConfig.Annotations[failpointCNIConfPathKey] = fpFilename
}

func injectShimFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, methodFps map[string]string) {
	for method, fp := range methodFps {
		_, err := failpoint.NewFailpoint(method, fp)
		require.NoError(t, err, "check failpoint %s for shim method %s", fp, method)

		sbConfig.Annotations[failpointShimPrefixKey+method] = fp
	}
}