diff --git a/internal/cri/server/container_create.go b/internal/cri/server/container_create.go index a3864be214..17cc9deac4 100644 --- a/internal/cri/server/container_create.go +++ b/internal/cri/server/container_create.go @@ -78,6 +78,15 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta if sandbox.Status.Get().State != sandboxstore.StateReady { return nil, fmt.Errorf("sandbox container %q is not running", sandboxID) } + // A sandbox without a process (pid 0) describes the namespaces and files it + // holds in the spec it returned on start; containers are wired up from it. + var sandboxSpec *runtimespec.Spec + if sandboxPid == 0 { + sandboxSpec, err = c.sandboxSpec(ctx, sandboxID) + if err != nil { + return nil, err + } + } span.SetAttributes( tracing.Attribute("sandbox.id", sandboxID), tracing.Attribute("sandbox.pid", sandboxPid), @@ -153,6 +162,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta podSandboxConfig: sandboxConfig, sandboxRuntimeHandler: sandbox.Metadata.RuntimeHandler, sandboxPid: sandboxPid, + sandboxSpec: sandboxSpec, NetNSPath: sandbox.NetNSPath, containerName: containerName, containerdImage: &containerdImage, @@ -183,6 +193,10 @@ type createContainerRequest struct { containerdImage *containerd.Image meta *containerstore.Metadata start time.Time + // sandboxSpec is the spec returned by the sandbox controller on start. It is + // only loaded for sandboxes without a process (sandboxPid == 0), where it is + // the source of the pod namespace paths and shared file mounts. + sandboxSpec *runtimespec.Spec } func (c *criService) createContainer(r *createContainerRequest) (_ string, retErr error) { @@ -275,6 +289,7 @@ func (c *criService) createContainer(r *createContainerRequest) (_ string, retEr volumeMounts, ociRuntime, runtimeHandler, + r.sandboxSpec, ) if err != nil { return "", fmt.Errorf("failed to generate container %q spec: %w", r.containerID, err) @@ -612,6 +627,7 @@ func (c *criService) buildContainerSpec( extraMounts []*runtime.Mount, ociRuntime criconfig.Runtime, runtimeHandler *runtime.RuntimeHandler, + sandboxSpec *runtimespec.Spec, ) (_ *runtimespec.Spec, retErr error) { var ( specOpts []oci.SpecOpts @@ -627,12 +643,13 @@ func (c *criService) buildContainerSpec( case isLinux: // Generate container mounts. // No mounts are passed for other platforms. - linuxMounts := c.linuxContainerMounts(sandboxID, config) + linuxMounts := c.linuxContainerMounts(sandboxID, config, sandboxSpec) specOpts, err = c.buildLinuxSpec( id, sandboxID, sandboxPid, + sandboxSpec, containerName, imageName, config, @@ -680,6 +697,7 @@ func (c *criService) buildLinuxSpec( id string, sandboxID string, sandboxPid uint32, + sandboxSpec *runtimespec.Spec, containerName string, imageName string, config *runtime.ContainerConfig, @@ -922,7 +940,7 @@ func (c *criService) buildLinuxSpec( specOpts = append(specOpts, customopts.WithOOMScoreAdj(config, c.config.RestrictOOMScoreAdj), - customopts.WithPodNamespaces(securityContext, sandboxPid, targetPid, uids, gids), + customopts.WithPodNamespacePaths(securityContext, sandboxConfig.GetLinux().GetSecurityContext().GetNamespaceOptions(), podNamespacePaths(sandboxPid, targetPid, sandboxSpec), uids, gids), customopts.WithSupplementalGroups(supplementalGroups), ) specOpts = append( @@ -1084,9 +1102,94 @@ func (c *criService) buildDarwinSpec( return specOpts, nil } +// sandboxSpec loads the OCI spec the sandbox controller returned when the +// sandbox was started. It is nil when the controller returned none. +func (c *criService) sandboxSpec(ctx context.Context, sandboxID string) (*runtimespec.Spec, error) { + sandboxInfo, err := c.client.SandboxStore().Get(ctx, sandboxID) + if err != nil { + return nil, fmt.Errorf("failed to get sandbox %q from metadata store: %w", sandboxID, err) + } + if sandboxInfo.Spec == nil { + return nil, nil + } + var spec runtimespec.Spec + if err := typeurl.UnmarshalTo(sandboxInfo.Spec, &spec); err != nil { + return nil, fmt.Errorf("failed to unmarshal spec of sandbox %q: %w", sandboxID, err) + } + return &spec, nil +} + +// podNamespacePaths resolves the namespaces a container joins. +// +// A PID-backed sandbox (pause) is a member of every pod namespace, so the +// paths are derived from its pid exactly as before. A sandbox without a +// process holds the pod namespaces itself and lists their paths in the Linux +// section of the spec it returned on start; a namespace missing from that +// spec is one the sandbox does not hold, which WithPodNamespacePaths turns into +// the host namespace when the pod asked for it and into an error otherwise. +// +// A sandbox that reports no process and returns no spec (or none with a Linux +// section) is a pre-existing Sandbox API shim that describes neither (VM based +// shims ignore the paths CRI generates); those keep the historical pid derived +// construction so that their behavior does not change. +func podNamespacePaths(sandboxPid, targetPid uint32, sandboxSpec *runtimespec.Spec) customopts.PodNamespacePaths { + if sandboxPid != 0 || sandboxSpec == nil || sandboxSpec.Linux == nil { + return customopts.PodNamespacePathsFromPid(sandboxPid, targetPid) + } + var paths customopts.PodNamespacePaths + for _, ns := range sandboxSpec.Linux.Namespaces { + switch ns.Type { + case runtimespec.NetworkNamespace: + paths.Network = ns.Path + case runtimespec.IPCNamespace: + paths.IPC = ns.Path + case runtimespec.UTSNamespace: + paths.UTS = ns.Path + case runtimespec.PIDNamespace: + // A PID namespace entry without a path is the sandbox declaring + // that it holds none and that each container gets a new one. + paths.PID = ns.Path + paths.PrivatePID = ns.Path == "" + case runtimespec.UserNamespace: + paths.User = ns.Path + } + } + // A container targeting another container's PID namespace joins that + // container, not the sandbox. + if targetPid != sandboxPid { + paths.PID = customopts.GetPIDNamespace(targetPid) + } + return paths +} + +// sandboxSharedFile returns the host path of a pod shared file, i.e. a file +// the sandbox provides to every container of the pod at destination (for +// example /etc/hosts). The fixed location under the CRI sandbox directory +// wins when it exists; otherwise the mount the sandbox declared for that +// destination in its spec is used, which is how a sandbox implementation that +// owns the files, such as a Sandbox API shim, hands them over. The result is +// empty when neither exists. +func (c *criService) sandboxSharedFile(fixedPath, destination string, sandboxSpec *runtimespec.Spec) string { + if _, err := c.os.Stat(fixedPath); err == nil { + return fixedPath + } + if sandboxSpec == nil { + return "" + } + for _, m := range sandboxSpec.Mounts { + if m.Destination != destination || m.Source == "" { + continue + } + if _, err := c.os.Stat(m.Source); err == nil { + return m.Source + } + } + return "" +} + // linuxContainerMounts sets up necessary container system file mounts // including /dev/shm, /etc/hosts and /etc/resolv.conf. -func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.ContainerConfig) []*runtime.Mount { +func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.ContainerConfig, sandboxSpec *runtimespec.Spec) []*runtime.Mount { var mounts []*runtime.Mount securityContext := config.GetLinux().GetSecurityContext() var uidMappings, gidMappings []*runtime.IDMapping @@ -1101,8 +1204,7 @@ func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.Cont // do not mount this in that case. // TODO(random-liu): Remove the check and always mount this when // containerd 1.1 and 1.2 are deprecated. - hostpath := c.getSandboxHostname(sandboxID) - if _, err := c.os.Stat(hostpath); err == nil { + if hostpath := c.sandboxSharedFile(c.getSandboxHostname(sandboxID), etcHostname, sandboxSpec); hostpath != "" { mounts = append(mounts, &runtime.Mount{ ContainerPath: etcHostname, HostPath: hostpath, @@ -1115,10 +1217,9 @@ func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.Cont } if !isInCRIMounts(etcHosts, config.GetMounts()) { - hostpath := c.getSandboxHosts(sandboxID) // /etc/hosts could be delegated to remote sandbox controller. That file isn't required to be existed // in host side for some sandbox runtimes. Skip it if we don't need it. - if _, err := c.os.Stat(hostpath); err == nil { + if hostpath := c.sandboxSharedFile(c.getSandboxHosts(sandboxID), etcHosts, sandboxSpec); hostpath != "" { mounts = append(mounts, &runtime.Mount{ ContainerPath: etcHosts, HostPath: hostpath, @@ -1133,10 +1234,9 @@ func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.Cont // Mount sandbox resolv.config. // TODO: Need to figure out whether we should always mount it as read-only if !isInCRIMounts(resolvConfPath, config.GetMounts()) { - hostpath := c.getResolvPath(sandboxID) // The ownership of /etc/resolv.conf could be delegated to remote sandbox controller. That file isn't // required to be existed in host side for some sandbox runtimes. Skip it if we don't need it. - if _, err := c.os.Stat(hostpath); err == nil { + if hostpath := c.sandboxSharedFile(c.getResolvPath(sandboxID), resolvConfPath, sandboxSpec); hostpath != "" { mounts = append(mounts, &runtime.Mount{ ContainerPath: resolvConfPath, HostPath: hostpath, @@ -1152,6 +1252,8 @@ func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.Cont sandboxDevShm := c.getSandboxDevShm(sandboxID) if securityContext.GetNamespaceOptions().GetIpc() == runtime.NamespaceMode_NODE { sandboxDevShm = devShm + } else if source := c.sandboxSharedFile(sandboxDevShm, devShm, sandboxSpec); source != "" { + sandboxDevShm = source } // The ownership of /dev/shm could be delegated to remote sandbox controller. That file isn't required // to be existed in host side for some sandbox runtimes. Skip it if we don't need it. diff --git a/internal/cri/server/container_create_linux_test.go b/internal/cri/server/container_create_linux_test.go index aa23fd81ac..271a215e72 100644 --- a/internal/cri/server/container_create_linux_test.go +++ b/internal/cri/server/container_create_linux_test.go @@ -251,7 +251,7 @@ func TestContainerCapabilities(t *testing.T) { c.allCaps = allCaps containerConfig.Linux.SecurityContext.Capabilities = test.capability - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) if selinux.GetEnabled() { @@ -286,7 +286,7 @@ func TestContainerSpecTty(t *testing.T) { c := newTestCRIService() for _, tty := range []bool{true, false} { containerConfig.Tty = tty - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) assert.Equal(t, tty, spec.Process.Terminal) @@ -313,7 +313,7 @@ func TestContainerSpecDefaultPath(t *testing.T) { imageConfig.Env = append(imageConfig.Env, pathenv) expected = pathenv } - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) assert.Contains(t, spec.Process.Env, expected) @@ -330,7 +330,7 @@ func TestContainerSpecReadonlyRootfs(t *testing.T) { c := newTestCRIService() for _, readonly := range []bool{true, false} { containerConfig.Linux.SecurityContext.ReadonlyRootfs = readonly - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) assert.Equal(t, readonly, spec.Root.Readonly) @@ -364,7 +364,7 @@ func TestContainerSpecWithExtraMounts(t *testing.T) { Readonly: false, }, } - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, extraMounts, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, extraMounts, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) var mounts, sysMounts []runtimespec.Mount @@ -430,7 +430,7 @@ func TestContainerAndSandboxPrivileged(t *testing.T) { sandboxConfig.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ Privileged: test.sandboxPrivileged, } - _, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + _, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) if test.expectError { assert.Error(t, err) } else { @@ -470,7 +470,7 @@ func TestPrivilegedBindMount(t *testing.T) { containerConfig.Linux.SecurityContext.Privileged = test.privileged sandboxConfig.Linux.SecurityContext.Privileged = test.privileged - spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) if test.expectedSysFSRO { @@ -542,7 +542,7 @@ func TestCgroupNamespace(t *testing.T) { containerConfig.Linux.SecurityContext.Privileged = tt.privileged sandboxConfig.Linux.SecurityContext.Privileged = tt.privileged - spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) hasCgroupNS := false @@ -711,7 +711,7 @@ func TestPidNamespace(t *testing.T) { } { t.Run(test.desc, func(t *testing.T) { containerConfig.Linux.SecurityContext.NamespaceOptions = &runtime.NamespaceOption{Pid: test.pidNS} - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) assert.Contains(t, spec.Linux.Namespaces, test.expected) }) @@ -885,7 +885,7 @@ func TestUserNamespace(t *testing.T) { sandboxUserns = test.sandboxUserNS } sandboxConfig.Linux.SecurityContext.NamespaceOptions = &runtime.NamespaceOption{UsernsOptions: sandboxUserns} - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) if test.err { require.Error(t, err) @@ -915,7 +915,7 @@ func TestNoDefaultRunMount(t *testing.T) { ociRuntime := config.Runtime{} c := newTestCRIService() - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) for _, mount := range spec.Mounts { assert.NotEqual(t, "/run", mount.Destination) @@ -1008,7 +1008,7 @@ func TestMaskedAndReadonlyPaths(t *testing.T) { sandboxConfig.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ Privileged: test.privileged, } - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) if !test.privileged { // specCheck presumes an unprivileged container specCheck(t, testID, testSandboxID, testPid, spec) @@ -1060,7 +1060,7 @@ func TestHostname(t *testing.T) { sandboxConfig.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ NamespaceOptions: &runtime.NamespaceOption{Network: test.networkNs}, } - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) assert.Contains(t, spec.Process.Env, test.expectedEnv) @@ -1169,7 +1169,7 @@ additional-group-for-root:x:22222:root containerConfig.Linux.SecurityContext = test.securityContext imageConfig.User = test.imageConfigUser - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) spec.Root.Path = tempRootDir // simulating /etc/{passwd, group} @@ -1244,7 +1244,7 @@ func TestNonRootUserAndDevices(t *testing.T) { }, } - spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil) + spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil, nil) assert.NoError(t, err) assert.Equal(t, test.expectedDeviceUID, *spec.Linux.Devices[0].UID) @@ -1317,7 +1317,7 @@ func TestPrivilegedDevices(t *testing.T) { PrivilegedWithoutHostDevices: test.privilegedWithoutHostDevices, PrivilegedWithoutHostDevicesAllDevicesAllowed: test.privilegedWithoutHostDevicesAllDevicesAllowed, } - spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) hostDevicesRaw, err := oci.HostDevices() @@ -1372,7 +1372,7 @@ func TestBaseOCISpec(t *testing.T) { testPid := uint32(1234) containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData() - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) @@ -1700,7 +1700,7 @@ containerEdits: }, } { t.Run(test.description, func(t *testing.T) { - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) @@ -1816,7 +1816,7 @@ func TestUserNamespaceWithHostNetwork(t *testing.T) { }, } - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) require.NotNil(t, spec) diff --git a/internal/cri/server/container_create_sandboxspec_linux_test.go b/internal/cri/server/container_create_sandboxspec_linux_test.go new file mode 100644 index 0000000000..d71a5d17ef --- /dev/null +++ b/internal/cri/server/container_create_sandboxspec_linux_test.go @@ -0,0 +1,168 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package server + +import ( + "os" + "testing" + + runtimespec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + + customopts "github.com/containerd/containerd/v2/internal/cri/opts" + ostesting "github.com/containerd/containerd/v2/pkg/os/testing" +) + +// pathBackedSandboxSpec is what a sandbox without a process returns on start: +// the namespaces it holds and the pod shared files it owns. +func pathBackedSandboxSpec() *runtimespec.Spec { + return &runtimespec.Spec{ + Linux: &runtimespec.Linux{ + Namespaces: []runtimespec.LinuxNamespace{ + {Type: runtimespec.NetworkNamespace, Path: "/var/run/netns/cni-1"}, + {Type: runtimespec.IPCNamespace, Path: "/run/sandbox/ns/ipc"}, + {Type: runtimespec.UTSNamespace, Path: "/run/sandbox/ns/uts"}, + }, + }, + Mounts: []runtimespec.Mount{ + {Destination: "/etc/hostname", Type: "bind", Source: "/run/sandbox/hostname"}, + {Destination: "/etc/hosts", Type: "bind", Source: "/run/sandbox/hosts"}, + {Destination: "/etc/resolv.conf", Type: "bind", Source: "/run/sandbox/resolv.conf"}, + {Destination: "/dev/shm", Type: "bind", Source: "/run/sandbox/shm"}, + }, + } +} + +func TestPodNamespacePaths(t *testing.T) { + spec := pathBackedSandboxSpec() + for name, tc := range map[string]struct { + sandboxPid, targetPid uint32 + spec *runtimespec.Spec + expected customopts.PodNamespacePaths + }{ + "a sandbox process wins over the spec": { + sandboxPid: 42, targetPid: 42, spec: spec, + expected: customopts.PodNamespacePathsFromPid(42, 42), + }, + "no process and no spec is a legacy sandbox api shim": { + expected: customopts.PodNamespacePathsFromPid(0, 0), + }, + "no process and a spec without a linux section is legacy too": { + spec: &runtimespec.Spec{}, + expected: customopts.PodNamespacePathsFromPid(0, 0), + }, + "no process and a spec with namespaces is path backed": { + spec: spec, + expected: customopts.PodNamespacePaths{ + Network: "/var/run/netns/cni-1", + IPC: "/run/sandbox/ns/ipc", + UTS: "/run/sandbox/ns/uts", + }, + }, + "a path backed sandbox in the host namespaces provides none": { + spec: &runtimespec.Spec{Linux: &runtimespec.Linux{}}, + expected: customopts.PodNamespacePaths{}, + }, + "a pid namespace entry without a path declares private pid namespaces": { + spec: &runtimespec.Spec{Linux: &runtimespec.Linux{Namespaces: []runtimespec.LinuxNamespace{ + {Type: runtimespec.NetworkNamespace, Path: "/var/run/netns/cni-1"}, + {Type: runtimespec.PIDNamespace}, + }}}, + expected: customopts.PodNamespacePaths{Network: "/var/run/netns/cni-1", PrivatePID: true}, + }, + "a target container pid namespace is joined by pid": { + targetPid: 7, spec: spec, + expected: customopts.PodNamespacePaths{ + Network: "/var/run/netns/cni-1", + IPC: "/run/sandbox/ns/ipc", + UTS: "/run/sandbox/ns/uts", + PID: "/proc/7/ns/pid", + }, + }, + } { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, podNamespacePaths(tc.sandboxPid, tc.targetPid, tc.spec)) + }) + } +} + +func TestLinuxContainerMountsFromSandboxSpec(t *testing.T) { + const testSandboxID = "test-sandbox-id" + config := &runtime.ContainerConfig{ + Metadata: &runtime.ContainerMetadata{Name: "test-name", Attempt: 1}, + Linux: &runtime.LinuxContainerConfig{ + SecurityContext: &runtime.LinuxContainerSecurityContext{}, + }, + } + exists := func(paths ...string) func(string) (os.FileInfo, error) { + set := make(map[string]struct{}, len(paths)) + for _, p := range paths { + set[p] = struct{}{} + } + return func(p string) (os.FileInfo, error) { + if _, ok := set[p]; ok { + return nil, nil + } + return nil, os.ErrNotExist + } + } + sharedMount := func(containerPath, hostPath string) *runtime.Mount { + return &runtime.Mount{ContainerPath: containerPath, HostPath: hostPath, SelinuxRelabel: true} + } + + t.Run("the sandbox spec provides the pod shared files", func(t *testing.T) { + c := newTestCRIService() + c.os.(*ostesting.FakeOS).StatFn = exists("/run/sandbox/hostname", "/run/sandbox/hosts", "/run/sandbox/resolv.conf", "/run/sandbox/shm") + assert.Equal(t, []*runtime.Mount{ + sharedMount(etcHostname, "/run/sandbox/hostname"), + sharedMount(etcHosts, "/run/sandbox/hosts"), + sharedMount(resolvConfPath, "/run/sandbox/resolv.conf"), + sharedMount(devShm, "/run/sandbox/shm"), + }, c.linuxContainerMounts(testSandboxID, config, pathBackedSandboxSpec())) + }) + + t.Run("the fixed sandbox directory wins over the spec", func(t *testing.T) { + c := newTestCRIService() + c.os.(*ostesting.FakeOS).StatFn = exists(c.getSandboxHosts(testSandboxID), "/run/sandbox/hosts") + assert.Equal(t, []*runtime.Mount{ + sharedMount(etcHosts, c.getSandboxHosts(testSandboxID)), + }, c.linuxContainerMounts(testSandboxID, config, pathBackedSandboxSpec())) + }) + + t.Run("a spec source that does not exist is not mounted", func(t *testing.T) { + c := newTestCRIService() + c.os.(*ostesting.FakeOS).StatFn = exists() + assert.Empty(t, c.linuxContainerMounts(testSandboxID, config, pathBackedSandboxSpec())) + }) + + t.Run("host ipc uses the host shm", func(t *testing.T) { + c := newTestCRIService() + c.os.(*ostesting.FakeOS).StatFn = exists(devShm, "/run/sandbox/shm") + hostIPC := &runtime.ContainerConfig{ + Metadata: config.Metadata, + Linux: &runtime.LinuxContainerConfig{ + SecurityContext: &runtime.LinuxContainerSecurityContext{ + NamespaceOptions: &runtime.NamespaceOption{Ipc: runtime.NamespaceMode_NODE}, + }, + }, + } + assert.Equal(t, []*runtime.Mount{ + {ContainerPath: devShm, HostPath: devShm, SelinuxRelabel: false}, + }, c.linuxContainerMounts(testSandboxID, hostIPC, pathBackedSandboxSpec())) + }) +} diff --git a/internal/cri/server/container_create_test.go b/internal/cri/server/container_create_test.go index 227b78aeac..ff7cb0aea1 100644 --- a/internal/cri/server/container_create_test.go +++ b/internal/cri/server/container_create_test.go @@ -70,7 +70,7 @@ func TestGeneralContainerSpec(t *testing.T) { c := newTestCRIService() testSandboxID := "sandbox-id" testContainerName := "container-name" - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) require.NoError(t, err) specCheck(t, testID, testSandboxID, testPid, spec) } @@ -146,7 +146,7 @@ func TestPodAnnotationPassthroughContainerSpec(t *testing.T) { PodAnnotations: test.podAnnotations, } spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, - containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) assert.NotNil(t, spec) specCheck(t, testID, testSandboxID, testPid, spec) @@ -508,7 +508,7 @@ func TestContainerAnnotationPassthroughContainerSpec(t *testing.T) { ContainerAnnotations: test.containerAnnotations, } spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, - containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) assert.NoError(t, err) assert.NotNil(t, spec) specCheck(t, testID, testSandboxID, testPid, spec) @@ -759,7 +759,7 @@ func TestLinuxContainerMounts(t *testing.T) { } c := newTestCRIService() c.os.(*ostesting.FakeOS).StatFn = test.statFn - mounts := c.linuxContainerMounts(testSandboxID, config) + mounts := c.linuxContainerMounts(testSandboxID, config, nil) assert.Equal(t, test.expectedMounts, mounts, test.desc) }) } diff --git a/internal/cri/server/container_create_windows_test.go b/internal/cri/server/container_create_windows_test.go index 2a12d16237..5cfe1401eb 100644 --- a/internal/cri/server/container_create_windows_test.go +++ b/internal/cri/server/container_create_windows_test.go @@ -165,7 +165,7 @@ func TestContainerWindowsNetworkNamespace(t *testing.T) { c := newTestCRIService() containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData() - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil, nil) assert.NoError(t, err) assert.NotNil(t, spec) specCheck(t, testID, testSandboxID, testPid, spec) @@ -187,7 +187,7 @@ func TestMountCleanPath(t *testing.T) { ContainerPath: "c:/test/container-path", HostPath: "c:/test/host-path", }) - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil, nil) assert.NoError(t, err) assert.NotNil(t, spec) specCheck(t, testID, testSandboxID, testPid, spec) @@ -207,7 +207,7 @@ func TestMountNamedPipe(t *testing.T) { ContainerPath: `\\.\pipe\foo`, HostPath: `\\.\pipe\foo`, }) - spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil) + spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil, nil) assert.NoError(t, err) assert.NotNil(t, spec) specCheck(t, testID, testSandboxID, testPid, spec) @@ -258,7 +258,7 @@ func TestHostProcessRequirements(t *testing.T) { sandboxConfig.Windows.SecurityContext = &runtime.WindowsSandboxSecurityContext{ HostProcess: test.sandboxHostProcess, } - _, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil) + _, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil, nil) if test.expectError { assert.Error(t, err) } else { @@ -355,7 +355,7 @@ func TestEntrypointAndCmdForArgsEscaped(t *testing.T) { Args: test.args, Windows: &runtime.WindowsContainerConfig{}, } - runtimeSpec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil) + runtimeSpec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil, nil) assert.NoError(t, err) assert.NotNil(t, runtimeSpec)