diff --git a/man/systemd-vmspawn.xml b/man/systemd-vmspawn.xml index 15324863daa..6c8960a3bfb 100644 --- a/man/systemd-vmspawn.xml +++ b/man/systemd-vmspawn.xml @@ -111,9 +111,24 @@ If specified, the VM is run with a temporary snapshot of its file system that is removed - immediately when the VM terminates. Only works with currently. + immediately when the VM terminates. - Note that will not work with . + For the snapshot is a btrfs snapshot of the directory, or a + copy of it if the file system does not support snapshots, made next to the directory, or inside + it if it is a mount point. + + For the snapshot is an unnamed temporary file in the directory of + the image, after following symlinks. If the image is a block device or a deleted file, + systemd-vmspawn creates the file in either /var/tmp/ or the + path $TMPDIR, $TEMP or $TMP are set to. + systemd-vmspawn uses the same directory if it cannot create the file next to the + image, for example because the directory of the image is read-only. + Everything the VM writes to its root disk is stored in this + file until the VM terminates. The VM can therefore fill up the file system that contains the + file. + + Note that will not work with + . diff --git a/src/vmspawn/vmspawn.c b/src/vmspawn/vmspawn.c index fc3405b5695..d15606665bf 100644 --- a/src/vmspawn/vmspawn.c +++ b/src/vmspawn/vmspawn.c @@ -26,6 +26,7 @@ #include "bus-locator.h" #include "bus-util.h" #include "capability-util.h" +#include "chattr-util.h" #include "common-signal.h" #include "copy.h" #include "discover-image.h" @@ -53,7 +54,6 @@ #include "machine-credential.h" #include "machine-register.h" #include "main-func.h" -#include "memfd-util.h" #include "mkdir.h" #include "namespace-util.h" #include "netif-util.h" @@ -2319,10 +2319,9 @@ static int resolve_disk_driver(DiskType dt, const char *filename, DriveInfo *inf return 0; } -static int prepare_primary_drive(const char *runtime_dir, DriveInfos *drives) { +static int prepare_primary_drive(DriveInfos *drives) { int r; - assert(runtime_dir); assert(drives); if (!arg_image) @@ -2370,16 +2369,51 @@ static int prepare_primary_drive(const char *runtime_dir, DriveInfos *drives) { * as qcow2 via blockdev-create, so no filesystem path is needed. * Skip for read-only drives (e.g. CDROM) where overlays are not meaningful. */ if (arg_ephemeral && !FLAGS_SET(d->flags, QMP_DRIVE_READ_ONLY)) { - _cleanup_close_ int overlay_fd = open(runtime_dir, O_TMPFILE | O_RDWR | O_CLOEXEC, 0600); - if (overlay_fd < 0) { - if (!ERRNO_IS_NOT_SUPPORTED(errno)) - return log_error_errno(errno, "Failed to create ephemeral overlay in '%s': %m", runtime_dir); + _cleanup_close_ int overlay_fd = -EBADF; + _cleanup_free_ char *image_dir = NULL; + const char *overlay_dir = NULL; - /* Fallback to memfd if O_TMPFILE is not supported */ - overlay_fd = memfd_new("vmspawn-overlay"); + /* /dev/ is a devtmpfs. An overlay next to a block device would therefore use host memory. A + * memfd or a deleted file has no links, and therefore no directory to create the overlay in. + * fd_get_path() returns the path of the opened file. If arg_image is a symlink, the overlay is + * therefore created next to the symlink target. */ + if (S_ISREG(st.st_mode) && st.st_nlink > 0) { + _cleanup_free_ char *image_path = NULL; + r = fd_get_path(d->fd, &image_path); + if (r < 0) + return log_error_errno(r, "Failed to get path of '%s': %m", arg_image); + + r = path_extract_directory(image_path, &image_dir); + if (r < 0) + return log_error_errno(r, "Failed to extract directory from '%s': %m", image_path); + + overlay_fd = open_tmpfile_unlinkable(image_dir, O_RDWR|O_CLOEXEC); if (overlay_fd < 0) - return log_error_errno(overlay_fd, "Failed to create ephemeral overlay via memfd: %m"); + log_debug_errno(overlay_fd, + "Failed to create ephemeral overlay in '%s', using the directory for large temporary files instead: %m", + image_dir); + else + overlay_dir = image_dir; } + + if (overlay_fd < 0) { + r = var_tmp_dir(&overlay_dir); + if (r < 0) + return log_error_errno(r, "Failed to determine directory for large temporary files: %m"); + + overlay_fd = open_tmpfile_unlinkable(overlay_dir, O_RDWR|O_CLOEXEC); + if (overlay_fd < 0) + return log_error_errno(overlay_fd, "Failed to create ephemeral overlay in '%s': %m", overlay_dir); + } + + /* The guest makes small random writes to the overlay. On btrfs, small random writes fragment a + * copy-on-write file. FS_NOCOW_FL only takes effect while the file is still empty. */ + r = chattr_fd(overlay_fd, FS_NOCOW_FL, FS_NOCOW_FL); + if (r < 0) + log_debug_errno(r, "Failed to disable copy-on-write for ephemeral overlay, ignoring: %m"); + + log_debug("Created ephemeral overlay in '%s'.", overlay_dir); + d->overlay_fd = TAKE_FD(overlay_fd); d->flags |= QMP_DRIVE_NO_FLUSH; d->grow_to = arg_grow_image; @@ -2480,10 +2514,9 @@ static int assign_pcie_ports(MachineConfig *c) { return 0; } -static int prepare_device_info(const char *runtime_dir, MachineConfig *c) { +static int prepare_device_info(MachineConfig *c) { int r; - assert(runtime_dir); assert(c); DriveInfos *drives = &c->drives; @@ -2494,7 +2527,7 @@ static int prepare_device_info(const char *runtime_dir, MachineConfig *c) { if (!drives->drives) return log_oom(); - r = prepare_primary_drive(runtime_dir, drives); + r = prepare_primary_drive(drives); if (r < 0) return r; @@ -3824,7 +3857,7 @@ static int run_virtual_machine(int kvm_device_fd, int vhost_device_fd) { child_pty = safe_close(child_pty); bridge_fds[1] = safe_close(bridge_fds[1]); - r = prepare_device_info(runtime_dir, &config); + r = prepare_device_info(&config); if (r < 0) return r; diff --git a/test/units/TEST-87-AUX-UTILS-VM.vmspawn-drives.sh b/test/units/TEST-87-AUX-UTILS-VM.vmspawn-drives.sh index 1e962e3d28e..a833d01ca4b 100755 --- a/test/units/TEST-87-AUX-UTILS-VM.vmspawn-drives.sh +++ b/test/units/TEST-87-AUX-UTILS-VM.vmspawn-drives.sh @@ -63,10 +63,35 @@ at_exit() { [[ -n "${VMSPAWN_MULTI_PID:-}" ]] && kill "$VMSPAWN_MULTI_PID" 2>/dev/null && wait "$VMSPAWN_MULTI_PID" 2>/dev/null [[ -n "${VMSPAWN_EPHEMERAL_PID:-}" ]] && kill "$VMSPAWN_EPHEMERAL_PID" 2>/dev/null && wait "$VMSPAWN_EPHEMERAL_PID" 2>/dev/null [[ -n "${VMSPAWN_GROW_PID:-}" ]] && kill "$VMSPAWN_GROW_PID" 2>/dev/null && wait "$VMSPAWN_GROW_PID" 2>/dev/null + mountpoint -q "$WORKDIR/ro" && umount "$WORKDIR/ro" rm -rf "$WORKDIR" } trap at_exit EXIT +# Prints the path of QEMU's fd for the ephemeral overlay of the given machine, if the overlay is in the given +# directory. The overlay is unlinked, so QEMU's fd is the only way to reach it. vmspawn registers QEMU itself +# as the machine leader, so machined knows its PID. +find_overlay_fd() { + local machine="${1:?}" dir="${2:?}" pid fd target + + pid="$(varlinkctl call /run/systemd/machine/io.systemd.Machine \ + io.systemd.Machine.List "{\"name\":\"$machine\"}" | jq -r '.leader.pid')" + + for fd in /proc/"$pid"/fd/*; do + target="$(readlink "$fd" 2>/dev/null)" || continue + # The overlay is an O_TMPFILE. On file systems without O_TMPFILE, vmspawn creates a named + # temporary file instead and unlinks it right away. + if [[ "$target" == "$dir"/#*" (deleted)" || "$target" == "$dir"/systemd-tmp-*" (deleted)" ]]; then + echo "$fd" + return 0 + fi + done + + echo "No ephemeral overlay in $dir found among the fds of QEMU:" >&2 + ls -l /proc/"$pid"/fd >&2 + return 1 +} + # Create a minimal root filesystem directory, then bake it into a raw ext4 image. # The guest doesn't need to fully boot — 'sleep infinity' keeps QEMU alive for QMP testing. mkdir -p "$WORKDIR/rootfs/sbin" @@ -144,13 +169,18 @@ echo "Multi-drive VM terminated cleanly" # background job, JOB_STATUS_CHANGE events are watched, and when the job # concludes the deferred continuation fires blockdev-add (overlay format) + # device_add. If any step fails, the root drive is never attached and the kernel -# panics — vmspawn exits without registering. +# panics — vmspawn exits without registering. The image is in a read-only +# directory, so the overlay has to be created in $TMPDIR instead. + +mkdir "$WORKDIR/ro" "$WORKDIR/large-tmp" +ln "$WORKDIR/root.raw" "$WORKDIR/ro/root.raw" +mount --bind -o ro "$WORKDIR/ro" "$WORKDIR/ro" MACHINE_EPHEMERAL="test-vmspawn-ephemeral-$$" -systemd-vmspawn \ +TMPDIR="$WORKDIR/large-tmp" systemd-vmspawn \ --machine="$MACHINE_EPHEMERAL" \ --ram=256M \ - --image="$WORKDIR/root.raw" \ + --image="$WORKDIR/ro/root.raw" \ --ephemeral \ --linux="$KERNEL" \ --tpm=no \ @@ -177,6 +207,9 @@ if grep -E '(add-fd|blockdev-add|blockdev-create|device_add|getfd|netdev_add|cha fi echo "No QMP device setup errors in ephemeral log" +OVERLAY="$(find_overlay_fd "$MACHINE_EPHEMERAL" "$WORKDIR/large-tmp")" +echo "Ephemeral overlay was created in \$TMPDIR and is QEMU fd ${OVERLAY##*/}" + machinectl terminate "$MACHINE_EPHEMERAL" timeout 10 bash -c "while machinectl status '$MACHINE_EPHEMERAL' &>/dev/null; do sleep .5; done" timeout 10 bash -c "while kill -0 '$VMSPAWN_EPHEMERAL_PID' 2>/dev/null; do sleep .5; done" @@ -185,7 +218,11 @@ echo "Ephemeral VM terminated cleanly" # --- Test 3: Ephemeral overlay grown with --grow-image= --- # In ephemeral mode the requested size has to land on the qcow2 overlay. The # image passed to --image= is opened read-only and has to come out of the run -# at its original size. +# at its original size. The image is passed through a symlink in another +# directory, and the overlay has to be created next to the symlink target. + +mkdir "$WORKDIR/links" +ln -s "$WORKDIR/root.raw" "$WORKDIR/links/root.raw" MACHINE_GROW="test-vmspawn-grow-$$" GROW_SIZE=$((512 * 1024 * 1024)) @@ -194,7 +231,7 @@ IMAGE_SIZE="$(stat -c %s "$WORKDIR/root.raw")" systemd-vmspawn \ --machine="$MACHINE_GROW" \ --ram=256M \ - --image="$WORKDIR/root.raw" \ + --image="$WORKDIR/links/root.raw" \ --ephemeral \ --grow-image="$GROW_SIZE" \ --linux="$KERNEL" \ @@ -210,21 +247,8 @@ echo "Grown ephemeral machine '$MACHINE_GROW' registered with machined" assert_eq "$(stat -c %s "$WORKDIR/root.raw")" "$IMAGE_SIZE" echo "Image passed to --image= was left at its original size" -# The overlay is unlinked, so QEMU's fd is the only way to reach it. vmspawn -# registers QEMU itself as the machine leader, so machined knows its PID. -QEMU_PID="$(varlinkctl call /run/systemd/machine/io.systemd.Machine \ - io.systemd.Machine.List "{\"name\":\"$MACHINE_GROW\"}" | jq -r '.leader.pid')" - -OVERLAY="" -for fd in /proc/"$QEMU_PID"/fd/*; do - target="$(readlink "$fd" 2>/dev/null)" || continue - # O_TMPFILE in the runtime directory, or the memfd fallback. - [[ "$target" == */vmspawn/"$MACHINE_GROW"/#* || "$target" == /memfd:vmspawn-overlay* ]] || continue - OVERLAY="$fd" - break -done -assert_neq "$OVERLAY" "" -echo "Ephemeral overlay is QEMU fd ${OVERLAY##*/}" +OVERLAY="$(find_overlay_fd "$MACHINE_GROW" "$WORKDIR")" +echo "Ephemeral overlay was created next to the image and is QEMU fd ${OVERLAY##*/}" # qcow2 header: the magic, followed by the virtual size as a big endian u64 at # offset 24. Read it out of the header directly, qemu-img may not be installed.