From 8a030d6537e42194cca894ebf89556af09dfade8 Mon Sep 17 00:00:00 2001 From: Henry Wang Date: Wed, 13 Nov 2024 19:27:49 +0000 Subject: [PATCH] Update overlay snapshotter to support multiple uid/gid mappings Signed-off-by: Henry Wang --- plugins/snapshots/overlay/overlay.go | 63 +-- plugins/snapshots/overlay/overlay_test.go | 449 ++++++++++++++-------- 2 files changed, 317 insertions(+), 195 deletions(-) diff --git a/plugins/snapshots/overlay/overlay.go b/plugins/snapshots/overlay/overlay.go index a0ea673ae2..80535ba823 100644 --- a/plugins/snapshots/overlay/overlay.go +++ b/plugins/snapshots/overlay/overlay.go @@ -29,6 +29,7 @@ import ( "github.com/containerd/containerd/v2/core/mount" "github.com/containerd/containerd/v2/core/snapshots" "github.com/containerd/containerd/v2/core/snapshots/storage" + "github.com/containerd/containerd/v2/internal/userns" "github.com/containerd/containerd/v2/plugins/snapshots/overlay/overlayutils" "github.com/containerd/continuity/fs" "github.com/containerd/log" @@ -424,41 +425,6 @@ func (o *snapshotter) getCleanupDirectories(ctx context.Context) ([]string, erro return cleanup, nil } -func validateIDMapping(mapping string) error { - var ( - hostID int - ctrID int - length int - ) - - if _, err := fmt.Sscanf(mapping, "%d:%d:%d", &ctrID, &hostID, &length); err != nil { - return err - } - // Almost impossible, but snapshots.WithLabels doesn't check it - if ctrID < 0 || hostID < 0 || length < 0 { - return fmt.Errorf("invalid mapping \"%d:%d:%d\"", ctrID, hostID, length) - } - if ctrID != 0 { - return fmt.Errorf("container mapping of 0 is only supported") - } - return nil -} - -func hostID(mapping string) (int, error) { - var ( - hostID int - ctrID int - length int - ) - if err := validateIDMapping(mapping); err != nil { - return -1, fmt.Errorf("invalid mapping: %w", err) - } - if _, err := fmt.Sscanf(mapping, "%d:%d:%d", &ctrID, &hostID, &length); err != nil { - return -1, err - } - return hostID, nil -} - func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt) (_ []mount.Mount, err error) { var ( s storage.Snapshot @@ -499,22 +465,35 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k return fmt.Errorf("failed to get snapshot info: %w", err) } - mappedUID := -1 - mappedGID := -1 + var ( + mappedUID, mappedGID = -1, -1 + uidmapLabel, gidmapLabel string + needsRemap = false + ) // NOTE: if idmapped mounts' supported by hosted kernel there may be // no parents at all, so overlayfs will not work and snapshotter // will use bind mount. To be able to create file objects inside the // rootfs -- just chown this only bound directory according to provided // {uid,gid}map. In case of one/multiple parents -- chown upperdir. if v, ok := info.Labels[snapshots.LabelSnapshotUIDMapping]; ok { - if mappedUID, err = hostID(v); err != nil { - return fmt.Errorf("failed to parse UID mapping: %w", err) - } + uidmapLabel = v + needsRemap = true } if v, ok := info.Labels[snapshots.LabelSnapshotGIDMapping]; ok { - if mappedGID, err = hostID(v); err != nil { - return fmt.Errorf("failed to parse GID mapping: %w", err) + gidmapLabel = v + needsRemap = true + } + + if needsRemap { + var idMap userns.IDMap + if err = idMap.Unmarshal(uidmapLabel, gidmapLabel); err != nil { + return fmt.Errorf("failed to unmarshal snapshot ID mapped labels: %w", err) } + root, err := idMap.RootPair() + if err != nil { + return fmt.Errorf("failed to find root pair: %w", err) + } + mappedUID, mappedGID = int(root.Uid), int(root.Gid) } if mappedUID == -1 || mappedGID == -1 { diff --git a/plugins/snapshots/overlay/overlay_test.go b/plugins/snapshots/overlay/overlay_test.go index 5cb6bc30a4..08a8c83d7c 100644 --- a/plugins/snapshots/overlay/overlay_test.go +++ b/plugins/snapshots/overlay/overlay_test.go @@ -31,6 +31,7 @@ import ( "github.com/containerd/containerd/v2/core/snapshots" "github.com/containerd/containerd/v2/core/snapshots/storage" "github.com/containerd/containerd/v2/core/snapshots/testsuite" + "github.com/containerd/containerd/v2/internal/userns" "github.com/containerd/containerd/v2/pkg/testutil" "github.com/containerd/containerd/v2/plugins/snapshots/overlay/overlayutils" "github.com/opencontainers/runtime-spec/specs-go" @@ -199,68 +200,280 @@ func testOverlayOverlayMount(t *testing.T, newSnapshotter testsuite.SnapshotterF } func testOverlayRemappedBind(t *testing.T, newSnapshotter testsuite.SnapshotterFunc) { - var ( - opts []snapshots.Opt - mounts []mount.Mount - ) + for _, test := range []struct { + idMap userns.IDMap + snapOpt func(idMap userns.IDMap) snapshots.Opt + expUID uint32 + expGID uint32 + expUIDMntOpt string + expGIDMntOpt string + }{ + { + idMap: userns.IDMap{ + UidMap: []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: 666, + Size: 65536, + }, + }, + GidMap: []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: 666, + Size: 65536, + }, + }, + }, + snapOpt: func(idMap userns.IDMap) snapshots.Opt { + return containerd.WithRemapperLabels( + idMap.UidMap[0].ContainerID, idMap.UidMap[0].HostID, + idMap.GidMap[0].ContainerID, idMap.GidMap[0].HostID, + idMap.UidMap[0].Size, + ) + }, + expUID: 666, + expGID: 666, + expUIDMntOpt: "uidmap=0:666:65536", + expGIDMntOpt: "gidmap=0:666:65536", + }, + { + idMap: userns.IDMap{ + UidMap: []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: 666, + Size: 1000, + }, + { + ContainerID: 1000, + HostID: 6666, + Size: 64536, + }, + }, + GidMap: []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: 888, + Size: 1000, + }, + { + ContainerID: 1000, + HostID: 8888, + Size: 64536, + }, + }, + }, + snapOpt: func(idMap userns.IDMap) snapshots.Opt { + return containerd.WithUserNSRemapperLabels(idMap.UidMap, idMap.GidMap) + }, + expUID: 666, + expGID: 888, + expUIDMntOpt: "uidmap=0:666:1000,1000:6666:64536", + expGIDMntOpt: "gidmap=0:888:1000,1000:8888:64536", + }, + } { + var ( + opts []snapshots.Opt + mounts []mount.Mount + ) - ctx := context.TODO() - root := t.TempDir() - o, _, err := newSnapshotter(ctx, root) - if err != nil { - t.Fatal(err) + ctx := context.TODO() + root := t.TempDir() + o, _, err := newSnapshotter(ctx, root) + if err != nil { + t.Fatal(err) + } + + if sn, ok := o.(*snapshotter); !ok || !sn.remapIDs { + t.Skip("overlayfs doesn't support idmapped mounts") + } + + opts = append(opts, test.snapOpt(test.idMap)) + + key := "/tmp/test" + if mounts, err = o.Prepare(ctx, key, "", opts...); err != nil { + t.Fatal(err) + } + + bp := getBasePath(ctx, o, root, key) + expected := []string{test.expUIDMntOpt, test.expGIDMntOpt, "rw", "rbind"} + + checkMountOpts := func() { + if len(mounts) != 1 { + t.Errorf("should only have 1 mount but received %d", len(mounts)) + } + + if len(mounts[0].Options) != len(expected) { + t.Errorf("expected %d options, but received %d", len(expected), len(mounts[0].Options)) + } + + m := mounts[0] + for i, v := range expected { + if m.Options[i] != v { + t.Errorf("mount option %q is not valid, expected %q", m.Options[i], v) + } + } + + st, err := os.Stat(filepath.Join(bp, "fs")) + if err != nil { + t.Errorf("failed to stat %s", filepath.Join(bp, "fs")) + } + + if stat, ok := st.Sys().(*syscall.Stat_t); !ok { + t.Errorf("incompatible types after stat call: *syscall.Stat_t expected") + } else if stat.Uid != test.expUID || stat.Gid != test.expGID { + t.Errorf("bad mapping: expected {uid: %d, gid: %d}; real {uid: %d, gid: %d}", test.expUID, test.expGID, int(stat.Uid), int(stat.Gid)) + } + } + checkMountOpts() + + expected[2] = "ro" + if err = o.Commit(ctx, "base", key, opts...); err != nil { + t.Fatal(err) + } + if mounts, err = o.View(ctx, key, "base", opts...); err != nil { + t.Fatal(err) + } + bp = getBasePath(ctx, o, root, key) + checkMountOpts() + + key = "/tmp/test1" + if mounts, err = o.Prepare(ctx, key, ""); err != nil { + t.Fatal(err) + } + + bp = getBasePath(ctx, o, root, key) + + expected = expected[2:] + expected[0] = "rw" + + test.expUID = 0 + test.expGID = 0 + + checkMountOpts() } +} - if sn, ok := o.(*snapshotter); !ok || !sn.remapIDs { - t.Skip("overlayfs doesn't support idmapped mounts") - } +func testOverlayRemappedActive(t *testing.T, newSnapshotter testsuite.SnapshotterFunc) { + for _, test := range []struct { + idMap userns.IDMap + snapOpt func(idMap userns.IDMap) snapshots.Opt + expUID uint32 + expGID uint32 + expUIDMntOpt string + expGIDMntOpt string + }{ + { + idMap: userns.IDMap{ + UidMap: []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: 666, + Size: 65536, + }, + }, + GidMap: []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: 666, + Size: 65536, + }, + }, + }, + snapOpt: func(idMap userns.IDMap) snapshots.Opt { + return containerd.WithRemapperLabels( + idMap.UidMap[0].ContainerID, idMap.UidMap[0].HostID, + idMap.GidMap[0].ContainerID, idMap.GidMap[0].HostID, + idMap.UidMap[0].Size, + ) + }, + expUID: 666, + expGID: 666, + expUIDMntOpt: "uidmap=0:666:65536", + expGIDMntOpt: "gidmap=0:666:65536", + }, + { + idMap: userns.IDMap{ + UidMap: []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: 666, + Size: 1000, + }, + { + ContainerID: 1000, + HostID: 6666, + Size: 64536, + }, + }, + GidMap: []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: 888, + Size: 1000, + }, + { + ContainerID: 1000, + HostID: 8888, + Size: 64536, + }, + }, + }, + snapOpt: func(idMap userns.IDMap) snapshots.Opt { + return containerd.WithUserNSRemapperLabels(idMap.UidMap, idMap.GidMap) + }, + expUID: 666, + expGID: 888, + expUIDMntOpt: "uidmap=0:666:1000,1000:6666:64536", + expGIDMntOpt: "gidmap=0:888:1000,1000:8888:64536", + }, + } { + var ( + opts []snapshots.Opt + mounts []mount.Mount + ) - hostID := uint32(666) - contID := uint32(0) - length := uint32(65536) + ctx := context.TODO() + root := t.TempDir() + o, _, err := newSnapshotter(ctx, root) + if err != nil { + t.Fatal(err) + } - uidMap := specs.LinuxIDMapping{ - ContainerID: contID, - HostID: hostID, - Size: length, - } - gidMap := specs.LinuxIDMapping{ - ContainerID: contID, - HostID: hostID, - Size: length, - } - opts = append(opts, containerd.WithRemapperLabels( - uidMap.ContainerID, uidMap.HostID, - gidMap.ContainerID, gidMap.HostID, - length), - ) + if sn, ok := o.(*snapshotter); !ok || !sn.remapIDs { + t.Skip("overlayfs doesn't support idmapped mounts") + } - key := "/tmp/test" - if mounts, err = o.Prepare(ctx, key, "", opts...); err != nil { - t.Fatal(err) - } + opts = append(opts, test.snapOpt(test.idMap)) - bp := getBasePath(ctx, o, root, key) - expected := []string{ - fmt.Sprintf("uidmap=%d:%d:%d", uidMap.ContainerID, uidMap.HostID, uidMap.Size), - fmt.Sprintf("gidmap=%d:%d:%d", gidMap.ContainerID, gidMap.HostID, gidMap.Size), - "rw", - "rbind", - } + key := "/tmp/test" + if _, err = o.Prepare(ctx, key, "", opts...); err != nil { + t.Fatal(err) + } + if err = o.Commit(ctx, "base", key, opts...); err != nil { + t.Fatal(err) + } + if mounts, err = o.Prepare(ctx, key, "base", opts...); err != nil { + t.Fatal(err) + } - checkMountOpts := func() { if len(mounts) != 1 { t.Errorf("should only have 1 mount but received %d", len(mounts)) } - if len(mounts[0].Options) != len(expected) { - t.Errorf("expected %d options, but received %d", len(expected), len(mounts[0].Options)) + bp := getBasePath(ctx, o, root, key) + expected := []string{ + test.expUIDMntOpt, test.expGIDMntOpt, + fmt.Sprintf("workdir=%s", filepath.Join(bp, "work")), + fmt.Sprintf("upperdir=%s", filepath.Join(bp, "fs")), + fmt.Sprintf("lowerdir=%s", getParents(ctx, o, root, key)[0]), } m := mounts[0] for i, v := range expected { if m.Options[i] != v { - t.Errorf("mount option %q is not valid, expected %q", m.Options[i], v) + t.Errorf("mount option %q is invalid, expected %q", m.Options[i], v) } } @@ -268,118 +481,12 @@ func testOverlayRemappedBind(t *testing.T, newSnapshotter testsuite.SnapshotterF if err != nil { t.Errorf("failed to stat %s", filepath.Join(bp, "fs")) } - if stat, ok := st.Sys().(*syscall.Stat_t); !ok { t.Errorf("incompatible types after stat call: *syscall.Stat_t expected") - } else if stat.Uid != uidMap.HostID || stat.Gid != gidMap.HostID { - t.Errorf("bad mapping: expected {uid: %d, gid: %d}; real {uid: %d, gid: %d}", uidMap.HostID, gidMap.HostID, int(stat.Uid), int(stat.Gid)) + } else if stat.Uid != test.expUID || stat.Gid != test.expGID { + t.Errorf("bad mapping: expected {uid: %d, gid: %d}; received {uid: %d, gid: %d}", test.expUID, test.expGID, int(stat.Uid), int(stat.Gid)) } } - checkMountOpts() - - expected[2] = "ro" - if err = o.Commit(ctx, "base", key, opts...); err != nil { - t.Fatal(err) - } - if mounts, err = o.View(ctx, key, "base", opts...); err != nil { - t.Fatal(err) - } - bp = getBasePath(ctx, o, root, key) - checkMountOpts() - - key = "/tmp/test1" - if mounts, err = o.Prepare(ctx, key, ""); err != nil { - t.Fatal(err) - } - - bp = getBasePath(ctx, o, root, key) - - expected = expected[2:] - expected[0] = "rw" - - uidMap.HostID = 0 - gidMap.HostID = 0 - - checkMountOpts() -} - -func testOverlayRemappedActive(t *testing.T, newSnapshotter testsuite.SnapshotterFunc) { - var ( - opts []snapshots.Opt - mounts []mount.Mount - ) - - ctx := context.TODO() - root := t.TempDir() - o, _, err := newSnapshotter(ctx, root) - if err != nil { - t.Fatal(err) - } - - if sn, ok := o.(*snapshotter); !ok || !sn.remapIDs { - t.Skip("overlayfs doesn't support idmapped mounts") - } - - hostID := uint32(666) - contID := uint32(0) - length := uint32(65536) - - uidMap := specs.LinuxIDMapping{ - ContainerID: contID, - HostID: hostID, - Size: length, - } - gidMap := specs.LinuxIDMapping{ - ContainerID: contID, - HostID: hostID, - Size: length, - } - opts = append(opts, containerd.WithRemapperLabels( - uidMap.ContainerID, uidMap.HostID, - gidMap.ContainerID, gidMap.HostID, - length), - ) - - key := "/tmp/test" - if _, err = o.Prepare(ctx, key, "", opts...); err != nil { - t.Fatal(err) - } - if err = o.Commit(ctx, "base", key, opts...); err != nil { - t.Fatal(err) - } - if mounts, err = o.Prepare(ctx, key, "base", opts...); err != nil { - t.Fatal(err) - } - - if len(mounts) != 1 { - t.Errorf("should only have 1 mount but received %d", len(mounts)) - } - - bp := getBasePath(ctx, o, root, key) - expected := []string{ - fmt.Sprintf("uidmap=%d:%d:%d", uidMap.ContainerID, uidMap.HostID, uidMap.Size), - fmt.Sprintf("gidmap=%d:%d:%d", gidMap.ContainerID, gidMap.HostID, gidMap.Size), - fmt.Sprintf("workdir=%s", filepath.Join(bp, "work")), - fmt.Sprintf("upperdir=%s", filepath.Join(bp, "fs")), - fmt.Sprintf("lowerdir=%s", getParents(ctx, o, root, key)[0]), - } - - m := mounts[0] - for i, v := range expected { - if m.Options[i] != v { - t.Errorf("mount option %q is invalid, expected %q", m.Options[i], v) - } - } - - st, err := os.Stat(filepath.Join(bp, "fs")) - if err != nil { - t.Errorf("failed to stat %s", filepath.Join(bp, "fs")) - } - if stat, ok := st.Sys().(*syscall.Stat_t); !ok { - t.Errorf("incompatible types after stat call: *syscall.Stat_t expected") - } else if stat.Uid != uidMap.HostID || stat.Gid != gidMap.HostID { - t.Errorf("bad mapping: expected {uid: %d, gid: %d}; received {uid: %d, gid: %d}", uidMap.HostID, gidMap.HostID, int(stat.Uid), int(stat.Gid)) - } } func testOverlayRemappedInvalidMapping(t *testing.T, newSnapshotter testsuite.SnapshotterFunc) { @@ -414,6 +521,24 @@ func testOverlayRemappedInvalidMapping(t *testing.T, newSnapshotter testsuite.Sn snapshots.LabelSnapshotGIDMapping: "-666:-666:-666", }), }, + "WithLabels: negative UID in multiple mappings must fail": { + snapshots.WithLabels(map[string]string{ + snapshots.LabelSnapshotUIDMapping: "1:1:1,-1:-1:-2", + snapshots.LabelSnapshotGIDMapping: "0:0:66666", + }), + }, + "WithLabels: negative GID in multiple mappings must fail": { + snapshots.WithLabels(map[string]string{ + snapshots.LabelSnapshotUIDMapping: "0:0:66666", + snapshots.LabelSnapshotGIDMapping: "-1:-1:-2,6:6:6", + }), + }, + "WithLabels: negative GID/UID in multiple mappings must fail": { + snapshots.WithLabels(map[string]string{ + snapshots.LabelSnapshotUIDMapping: "-666:-666:-666,1:1:1", + snapshots.LabelSnapshotGIDMapping: "-666:-666:-666,2:2:2", + }), + }, "WithRemapperLabels: container ID (GID/UID) other than 0 must fail": { containerd.WithRemapperLabels(666, 666, 666, 666, 666), }, @@ -423,6 +548,24 @@ func testOverlayRemappedInvalidMapping(t *testing.T, newSnapshotter testsuite.Sn "WithRemapperLabels: container ID (GID) other than 0 must fail": { containerd.WithRemapperLabels(0, 0, 666, 0, 4294967295), }, + "WithUserNSRemapperLabels: container ID (GID/UID) other than 0 must fail": { + containerd.WithUserNSRemapperLabels( + []specs.LinuxIDMapping{{ContainerID: 666, HostID: 666, Size: 666}}, + []specs.LinuxIDMapping{{ContainerID: 666, HostID: 666, Size: 666}}, + ), + }, + "WithUserNSRemapperLabels: container ID (UID) other than 0 must fail": { + containerd.WithUserNSRemapperLabels( + []specs.LinuxIDMapping{{ContainerID: 666, HostID: 0, Size: 65536}}, + []specs.LinuxIDMapping{{ContainerID: 0, HostID: 0, Size: 65536}}, + ), + }, + "WithUserNSRemapperLabels: container ID (GID) other than 0 must fail": { + containerd.WithUserNSRemapperLabels( + []specs.LinuxIDMapping{{ContainerID: 0, HostID: 0, Size: 4294967295}}, + []specs.LinuxIDMapping{{ContainerID: 666, HostID: 0, Size: 4294967295}}, + ), + }, } { t.Log(desc) if _, err = o.Prepare(ctx, key, "", opts...); err == nil {