mirror of
https://github.com/containerd/containerd.git
synced 2026-06-24 08:48:48 +00:00
d7d7b10f99
In some spots we can get away with only reading a subset of the cgroup stats we are today. It would be reaaally nice for container stats in the cri plugin, but they're requested via the task API and we have no way to signify we only want a subset through this surface yet. We can still get some benefit in the stats collector and the existing sandbox stats where we only need mem and cpu. Signed-off-by: Danny Canter <danny@dcantah.dev>
356 lines
11 KiB
Go
356 lines
11 KiB
Go
//go:build linux
|
|
|
|
/*
|
|
Copyright The containerd Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package integration
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/containerd/cgroups/v3"
|
|
"github.com/containerd/cgroups/v3/cgroup1"
|
|
cgroupsv2 "github.com/containerd/cgroups/v3/cgroup2"
|
|
containerd "github.com/containerd/containerd/v2/client"
|
|
"github.com/containerd/containerd/v2/integration/images"
|
|
criopts "github.com/containerd/containerd/v2/internal/cri/opts"
|
|
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
|
|
)
|
|
|
|
func checkMemoryLimit(t *testing.T, spec *runtimespec.Spec, memLimit int64) {
|
|
require.NotNil(t, spec)
|
|
require.NotNil(t, spec.Linux)
|
|
require.NotNil(t, spec.Linux.Resources)
|
|
require.NotNil(t, spec.Linux.Resources.Memory)
|
|
require.NotNil(t, spec.Linux.Resources.Memory.Limit)
|
|
assert.Equal(t, memLimit, *spec.Linux.Resources.Memory.Limit)
|
|
}
|
|
|
|
func checkMemorySwapLimit(t *testing.T, spec *runtimespec.Spec, memLimit *int64) {
|
|
require.NotNil(t, spec)
|
|
require.NotNil(t, spec.Linux)
|
|
require.NotNil(t, spec.Linux.Resources)
|
|
require.NotNil(t, spec.Linux.Resources.Memory)
|
|
if memLimit == nil {
|
|
require.Nil(t, spec.Linux.Resources.Memory.Swap)
|
|
} else {
|
|
require.NotNil(t, spec.Linux.Resources.Memory.Swap)
|
|
assert.Equal(t, *memLimit, *spec.Linux.Resources.Memory.Swap)
|
|
}
|
|
}
|
|
|
|
func checkMemoryLimitInContainerStatus(t *testing.T, status *runtime.ContainerStatus, memLimit int64) {
|
|
t.Helper()
|
|
require.NotNil(t, status)
|
|
require.NotNil(t, status.Resources)
|
|
require.NotNil(t, status.Resources.Linux)
|
|
require.NotNil(t, status.Resources.Linux.MemoryLimitInBytes)
|
|
assert.Equal(t, memLimit, status.Resources.Linux.MemoryLimitInBytes)
|
|
}
|
|
|
|
func getCgroupSwapLimitForTask(t *testing.T, task containerd.Task) uint64 {
|
|
if cgroups.Mode() == cgroups.Unified {
|
|
groupPath, err := cgroupsv2.PidGroupPath(int(task.Pid()))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cgroup2, err := cgroupsv2.Load(groupPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stat, err := cgroup2.StatFiltered(cgroupsv2.StatMemory)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return stat.Memory.SwapLimit + stat.Memory.UsageLimit
|
|
}
|
|
cgroup, err := cgroup1.Load(cgroup1.PidPath(int(task.Pid())))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stat, err := cgroup.Stat(cgroup1.IgnoreNotExist)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return stat.Memory.HierarchicalSwapLimit
|
|
}
|
|
|
|
func getCgroupMemoryLimitForTask(t *testing.T, task containerd.Task) uint64 {
|
|
if cgroups.Mode() == cgroups.Unified {
|
|
groupPath, err := cgroupsv2.PidGroupPath(int(task.Pid()))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cgroup2, err := cgroupsv2.Load(groupPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stat, err := cgroup2.StatFiltered(cgroupsv2.StatMemory)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return stat.Memory.UsageLimit
|
|
}
|
|
|
|
cgroup, err := cgroup1.Load(cgroup1.PidPath(int(task.Pid())))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stat, err := cgroup.Stat(cgroup1.IgnoreNotExist)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return stat.Memory.Usage.Limit
|
|
}
|
|
|
|
func isSwapLikelyEnabled() bool {
|
|
// Check whether swap is enabled.
|
|
swapFile := "/proc/swaps"
|
|
swapData, err := os.ReadFile(swapFile)
|
|
if err != nil {
|
|
// We can't read the file or it doesn't exist, assume we don't have swap.
|
|
return false
|
|
}
|
|
|
|
swapData = bytes.TrimSpace(swapData) // extra trailing \n
|
|
swapLines := strings.Split(string(swapData), "\n")
|
|
|
|
// If there is more than one line (table headers) in /proc/swaps, swap is enabled
|
|
if len(swapLines) <= 1 {
|
|
return false
|
|
}
|
|
|
|
// Linux Kernel's prior to 5.8 can disable swap accounting and is disabled
|
|
// by default on Ubuntu. Most systems that run with cgroupsv2 enabled likely
|
|
// have swap accounting enabled, here we assume that is true when running with
|
|
// cgroupsv2 and check on cgroupsv1.
|
|
if cgroups.Mode() == cgroups.Unified {
|
|
return true
|
|
}
|
|
|
|
_, err = os.Stat("/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes")
|
|
// Assume any error means this test can't run for now.
|
|
return err == nil
|
|
}
|
|
|
|
func TestUpdateContainerResources_MemorySwap(t *testing.T) {
|
|
if !isSwapLikelyEnabled() {
|
|
t.Skipf("Swap or swap accounting are not enabled. Swap is required for this test")
|
|
return
|
|
}
|
|
|
|
t.Log("Create a sandbox")
|
|
sb, sbConfig := PodSandboxConfigWithCleanup(t, "sandbox", "update-container-swap-resources")
|
|
|
|
pauseImage := images.Get(images.Pause)
|
|
EnsureImageExists(t, pauseImage)
|
|
|
|
memoryLimit := int64(128 * 1024 * 1024)
|
|
baseSwapLimit := int64(200 * 1024 * 1024)
|
|
increasedSwapLimit := int64(256 * 1024 * 1024)
|
|
|
|
expectedBaseSwap := baseSwapLimit
|
|
expectedIncreasedSwap := increasedSwapLimit
|
|
|
|
t.Log("Create a container with memory limit but no swap")
|
|
cnConfig := ContainerConfig(
|
|
"container",
|
|
pauseImage,
|
|
WithResources(&runtime.LinuxContainerResources{
|
|
MemoryLimitInBytes: memoryLimit,
|
|
MemorySwapLimitInBytes: baseSwapLimit,
|
|
}),
|
|
)
|
|
cn, err := runtimeService.CreateContainer(sb, cnConfig, sbConfig)
|
|
require.NoError(t, err)
|
|
|
|
t.Log("Check memory limit in container OCI spec")
|
|
container, err := containerdClient.LoadContainer(context.Background(), cn)
|
|
require.NoError(t, err)
|
|
spec, err := container.Spec(context.Background())
|
|
require.NoError(t, err)
|
|
checkMemoryLimit(t, spec, memoryLimit)
|
|
checkMemorySwapLimit(t, spec, &expectedBaseSwap)
|
|
|
|
t.Log("Check memory limit in container OCI spec")
|
|
spec, err = container.Spec(context.Background())
|
|
require.NoError(t, err)
|
|
sw1 := baseSwapLimit
|
|
checkMemorySwapLimit(t, spec, &sw1)
|
|
|
|
t.Log("Start the container")
|
|
require.NoError(t, runtimeService.StartContainer(cn))
|
|
task, err := container.Task(context.Background(), nil)
|
|
require.NoError(t, err)
|
|
|
|
t.Log("Check memory limit in cgroup")
|
|
memLimit := getCgroupMemoryLimitForTask(t, task)
|
|
swapLimit := getCgroupSwapLimitForTask(t, task)
|
|
assert.Equal(t, uint64(memoryLimit), memLimit)
|
|
assert.Equal(t, uint64(expectedBaseSwap), swapLimit)
|
|
|
|
t.Log("Update container memory limit after started")
|
|
err = runtimeService.UpdateContainerResources(cn, &runtime.LinuxContainerResources{
|
|
MemorySwapLimitInBytes: increasedSwapLimit,
|
|
}, nil)
|
|
require.NoError(t, err)
|
|
|
|
t.Log("Check memory limit in container OCI spec")
|
|
spec, err = container.Spec(context.Background())
|
|
require.NoError(t, err)
|
|
checkMemorySwapLimit(t, spec, &increasedSwapLimit)
|
|
|
|
t.Log("Check memory limit in cgroup")
|
|
swapLimit = getCgroupSwapLimitForTask(t, task)
|
|
assert.Equal(t, uint64(expectedIncreasedSwap), swapLimit)
|
|
}
|
|
|
|
func TestUpdateContainerResources_MemoryLimit(t *testing.T) {
|
|
t.Log("Create a sandbox")
|
|
sb, sbConfig := PodSandboxConfigWithCleanup(t, "sandbox", "update-container-resources")
|
|
|
|
pauseImage := images.Get(images.Pause)
|
|
EnsureImageExists(t, pauseImage)
|
|
|
|
expectedSwapLimit := func(memoryLimit int64) *int64 {
|
|
if criopts.SwapControllerAvailable() {
|
|
return &memoryLimit
|
|
}
|
|
return nil
|
|
}
|
|
|
|
t.Log("Create a container with memory limit")
|
|
cnConfig := ContainerConfig(
|
|
"container",
|
|
pauseImage,
|
|
WithResources(&runtime.LinuxContainerResources{
|
|
MemoryLimitInBytes: 200 * 1024 * 1024,
|
|
}),
|
|
)
|
|
cn, err := runtimeService.CreateContainer(sb, cnConfig, sbConfig)
|
|
require.NoError(t, err)
|
|
|
|
t.Log("Check memory limit in container OCI spec")
|
|
container, err := containerdClient.LoadContainer(context.Background(), cn)
|
|
require.NoError(t, err)
|
|
spec, err := container.Spec(context.Background())
|
|
require.NoError(t, err)
|
|
checkMemoryLimit(t, spec, 200*1024*1024)
|
|
checkMemorySwapLimit(t, spec, expectedSwapLimit(200*1024*1024))
|
|
|
|
t.Log("Update container memory limit after created")
|
|
err = runtimeService.UpdateContainerResources(cn, &runtime.LinuxContainerResources{
|
|
MemoryLimitInBytes: 400 * 1024 * 1024,
|
|
}, nil)
|
|
require.NoError(t, err)
|
|
|
|
t.Log("Check memory limit in container OCI spec")
|
|
spec, err = container.Spec(context.Background())
|
|
require.NoError(t, err)
|
|
checkMemoryLimit(t, spec, 400*1024*1024)
|
|
checkMemorySwapLimit(t, spec, expectedSwapLimit(400*1024*1024))
|
|
|
|
t.Log("Start the container")
|
|
require.NoError(t, runtimeService.StartContainer(cn))
|
|
task, err := container.Task(context.Background(), nil)
|
|
require.NoError(t, err)
|
|
|
|
t.Log("Check memory limit in cgroup")
|
|
memLimit := getCgroupMemoryLimitForTask(t, task)
|
|
assert.Equal(t, uint64(400*1024*1024), memLimit)
|
|
if criopts.SwapControllerAvailable() {
|
|
swapLimit := getCgroupSwapLimitForTask(t, task)
|
|
assert.Equal(t, uint64(400*1024*1024), swapLimit)
|
|
}
|
|
|
|
t.Log("Update container memory limit after started")
|
|
err = runtimeService.UpdateContainerResources(cn, &runtime.LinuxContainerResources{
|
|
MemoryLimitInBytes: 800 * 1024 * 1024,
|
|
}, nil)
|
|
require.NoError(t, err)
|
|
|
|
t.Log("Check memory limit in container OCI spec")
|
|
spec, err = container.Spec(context.Background())
|
|
require.NoError(t, err)
|
|
checkMemoryLimit(t, spec, 800*1024*1024)
|
|
checkMemorySwapLimit(t, spec, expectedSwapLimit(800*1024*1024))
|
|
|
|
t.Log("Check memory limit in cgroup")
|
|
memLimit = getCgroupMemoryLimitForTask(t, task)
|
|
assert.Equal(t, uint64(800*1024*1024), memLimit)
|
|
if criopts.SwapControllerAvailable() {
|
|
swapLimit := getCgroupSwapLimitForTask(t, task)
|
|
assert.Equal(t, uint64(800*1024*1024), swapLimit)
|
|
}
|
|
|
|
}
|
|
|
|
func TestUpdateContainerResources_StatusUpdated(t *testing.T) {
|
|
t.Log("Create a sandbox")
|
|
sb, sbConfig := PodSandboxConfigWithCleanup(t, "sandbox", "update-container-resources")
|
|
|
|
pauseImage := images.Get(images.Pause)
|
|
EnsureImageExists(t, pauseImage)
|
|
|
|
t.Log("Create a container with memory limit")
|
|
cnConfig := ContainerConfig(
|
|
"container",
|
|
pauseImage,
|
|
WithResources(&runtime.LinuxContainerResources{
|
|
MemoryLimitInBytes: 200 * 1024 * 1024,
|
|
}),
|
|
)
|
|
cn, err := runtimeService.CreateContainer(sb, cnConfig, sbConfig)
|
|
require.NoError(t, err)
|
|
|
|
t.Log("Check memory limit in container status")
|
|
status, err := runtimeService.ContainerStatus(cn)
|
|
checkMemoryLimitInContainerStatus(t, status, 200*1024*1024)
|
|
require.NoError(t, err)
|
|
|
|
t.Log("Update container memory limit after created")
|
|
err = runtimeService.UpdateContainerResources(cn, &runtime.LinuxContainerResources{
|
|
MemoryLimitInBytes: 400 * 1024 * 1024,
|
|
}, nil)
|
|
require.NoError(t, err)
|
|
|
|
t.Log("Check memory limit in container status")
|
|
status, err = runtimeService.ContainerStatus(cn)
|
|
checkMemoryLimitInContainerStatus(t, status, 400*1024*1024)
|
|
require.NoError(t, err)
|
|
|
|
t.Log("Start the container")
|
|
require.NoError(t, runtimeService.StartContainer(cn))
|
|
|
|
t.Log("Update container memory limit after started")
|
|
err = runtimeService.UpdateContainerResources(cn, &runtime.LinuxContainerResources{
|
|
MemoryLimitInBytes: 800 * 1024 * 1024,
|
|
}, nil)
|
|
require.NoError(t, err)
|
|
|
|
t.Log("Check memory limit in container status")
|
|
status, err = runtimeService.ContainerStatus(cn)
|
|
checkMemoryLimitInContainerStatus(t, status, 800*1024*1024)
|
|
require.NoError(t, err)
|
|
}
|