vmspawn: put the ephemeral overlay next to the image

vmspawn creates the ephemeral overlay of an --image= in the runtime directory,
which is usually on tmpfs, and falls back to a memfd. The guest's writes
therefore use up host memory. Create the overlay next to the image after
following symlinks. nspawn puts its ephemeral copy of an image there too, and
vmspawn puts the snapshot for --directory= next to the directory.

If the image is a block device, use the directory for large temporary files
instead, because /dev/ is a devtmpfs. Do the same if the image is a deleted
file or a memfd, because a deleted file or a memfd has no directory. Also do
the same if the overlay cannot be created next to the image, for example
because the directory is read-only.

Disable copy-on-write on the overlay, because small random writes fragment a
copy-on-write file on btrfs.

man/systemd-vmspawn.xml

@@ -111,9 +111,24 @@
           <term><option>--ephemeral</option></term>
 
           <listitem><para>If specified, the VM is run with a temporary snapshot of its file system that is removed
-          immediately when the VM terminates. Only works with <option>--image=</option> currently.
+          immediately when the VM terminates.</para>
 
-          Note that <option>--ephemeral</option> will not work with <option>--extra-drive=</option>.</para>
+          <para>For <option>--directory=</option> the snapshot is a btrfs snapshot of the directory, or a
+          copy of it if the file system does not support snapshots, made next to the directory, or inside
+          it if it is a mount point.</para>
+
+          <para>For <option>--image=</option> the snapshot is an unnamed temporary file in the directory of
+          the image, after following symlinks. If the image is a block device or a deleted file,
+          <command>systemd-vmspawn</command> creates the file in either <filename>/var/tmp/</filename> or the
+          path <varname>$TMPDIR</varname>, <varname>$TEMP</varname> or <varname>$TMP</varname> are set to.
+          <command>systemd-vmspawn</command> uses the same directory if it cannot create the file next to the
+          image, for example because the directory of the image is read-only.
+          Everything the VM writes to its root disk is stored in this
+          file until the VM terminates. The VM can therefore fill up the file system that contains the
+          file.</para>
+
+          <para>Note that <option>--ephemeral</option> will not work with
+          <option>--extra-drive=</option>.</para>
 
           <xi:include href="version-info.xml" xpointer="v260"/></listitem>
         </varlistentry>

src/vmspawn/vmspawn.c

@@ -26,6 +26,7 @@
 #include "bus-locator.h"
 #include "bus-util.h"
 #include "capability-util.h"
+#include "chattr-util.h"
 #include "common-signal.h"
 #include "copy.h"
 #include "discover-image.h"
@@ -53,7 +54,6 @@
 #include "machine-credential.h"
 #include "machine-register.h"
 #include "main-func.h"
-#include "memfd-util.h"
 #include "mkdir.h"
 #include "namespace-util.h"
 #include "netif-util.h"
@@ -2319,10 +2319,9 @@ static int resolve_disk_driver(DiskType dt, const char *filename, DriveInfo *inf
         return 0;
 }
 
-static int prepare_primary_drive(const char *runtime_dir, DriveInfos *drives) {
+static int prepare_primary_drive(DriveInfos *drives) {
         int r;
 
-        assert(runtime_dir);
         assert(drives);
 
         if (!arg_image)
@@ -2370,16 +2369,51 @@ static int prepare_primary_drive(const char *runtime_dir, DriveInfos *drives) {
          * as qcow2 via blockdev-create, so no filesystem path is needed.
          * Skip for read-only drives (e.g. CDROM) where overlays are not meaningful. */
         if (arg_ephemeral && !FLAGS_SET(d->flags, QMP_DRIVE_READ_ONLY)) {
-                _cleanup_close_ int overlay_fd = open(runtime_dir, O_TMPFILE | O_RDWR | O_CLOEXEC, 0600);
-                if (overlay_fd < 0) {
-                        if (!ERRNO_IS_NOT_SUPPORTED(errno))
-                                return log_error_errno(errno, "Failed to create ephemeral overlay in '%s': %m", runtime_dir);
+                _cleanup_close_ int overlay_fd = -EBADF;
+                _cleanup_free_ char *image_dir = NULL;
+                const char *overlay_dir = NULL;
 
-                        /* Fallback to memfd if O_TMPFILE is not supported */
-                        overlay_fd = memfd_new("vmspawn-overlay");
+                /* /dev/ is a devtmpfs. An overlay next to a block device would therefore use host memory. A
+                 * memfd or a deleted file has no links, and therefore no directory to create the overlay in.
+                 * fd_get_path() returns the path of the opened file. If arg_image is a symlink, the overlay is
+                 * therefore created next to the symlink target. */
+                if (S_ISREG(st.st_mode) && st.st_nlink > 0) {
+                        _cleanup_free_ char *image_path = NULL;
+                        r = fd_get_path(d->fd, &image_path);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to get path of '%s': %m", arg_image);
+
+                        r = path_extract_directory(image_path, &image_dir);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to extract directory from '%s': %m", image_path);
+
+                        overlay_fd = open_tmpfile_unlinkable(image_dir, O_RDWR|O_CLOEXEC);
                         if (overlay_fd < 0)
-                                return log_error_errno(overlay_fd, "Failed to create ephemeral overlay via memfd: %m");
+                                log_debug_errno(overlay_fd,
+                                                "Failed to create ephemeral overlay in '%s', using the directory for large temporary files instead: %m",
+                                                image_dir);
+                        else
+                                overlay_dir = image_dir;
                 }
+
+                if (overlay_fd < 0) {
+                        r = var_tmp_dir(&overlay_dir);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to determine directory for large temporary files: %m");
+
+                        overlay_fd = open_tmpfile_unlinkable(overlay_dir, O_RDWR|O_CLOEXEC);
+                        if (overlay_fd < 0)
+                                return log_error_errno(overlay_fd, "Failed to create ephemeral overlay in '%s': %m", overlay_dir);
+                }
+
+                /* The guest makes small random writes to the overlay. On btrfs, small random writes fragment a
+                 * copy-on-write file. FS_NOCOW_FL only takes effect while the file is still empty. */
+                r = chattr_fd(overlay_fd, FS_NOCOW_FL, FS_NOCOW_FL);
+                if (r < 0)
+                        log_debug_errno(r, "Failed to disable copy-on-write for ephemeral overlay, ignoring: %m");
+
+                log_debug("Created ephemeral overlay in '%s'.", overlay_dir);
+
                 d->overlay_fd = TAKE_FD(overlay_fd);
                 d->flags |= QMP_DRIVE_NO_FLUSH;
                 d->grow_to = arg_grow_image;
@@ -2480,10 +2514,9 @@ static int assign_pcie_ports(MachineConfig *c) {
         return 0;
 }
 
-static int prepare_device_info(const char *runtime_dir, MachineConfig *c) {
+static int prepare_device_info(MachineConfig *c) {
         int r;
 
-        assert(runtime_dir);
         assert(c);
 
         DriveInfos *drives = &c->drives;
@@ -2494,7 +2527,7 @@ static int prepare_device_info(const char *runtime_dir, MachineConfig *c) {
         if (!drives->drives)
                 return log_oom();
 
-        r = prepare_primary_drive(runtime_dir, drives);
+        r = prepare_primary_drive(drives);
         if (r < 0)
                 return r;
 
@@ -3824,7 +3857,7 @@ static int run_virtual_machine(int kvm_device_fd, int vhost_device_fd) {
         child_pty = safe_close(child_pty);
         bridge_fds[1] = safe_close(bridge_fds[1]);
 
-        r = prepare_device_info(runtime_dir, &config);
+        r = prepare_device_info(&config);
         if (r < 0)
                 return r;
 

test/units/TEST-87-AUX-UTILS-VM.vmspawn-drives.sh

@@ -63,10 +63,35 @@ at_exit() {
     [[ -n "${VMSPAWN_MULTI_PID:-}" ]] && kill "$VMSPAWN_MULTI_PID" 2>/dev/null && wait "$VMSPAWN_MULTI_PID" 2>/dev/null
     [[ -n "${VMSPAWN_EPHEMERAL_PID:-}" ]] && kill "$VMSPAWN_EPHEMERAL_PID" 2>/dev/null && wait "$VMSPAWN_EPHEMERAL_PID" 2>/dev/null
     [[ -n "${VMSPAWN_GROW_PID:-}" ]] && kill "$VMSPAWN_GROW_PID" 2>/dev/null && wait "$VMSPAWN_GROW_PID" 2>/dev/null
+    mountpoint -q "$WORKDIR/ro" && umount "$WORKDIR/ro"
     rm -rf "$WORKDIR"
 }
 trap at_exit EXIT
 
+# Prints the path of QEMU's fd for the ephemeral overlay of the given machine, if the overlay is in the given
+# directory. The overlay is unlinked, so QEMU's fd is the only way to reach it. vmspawn registers QEMU itself
+# as the machine leader, so machined knows its PID.
+find_overlay_fd() {
+    local machine="${1:?}" dir="${2:?}" pid fd target
+
+    pid="$(varlinkctl call /run/systemd/machine/io.systemd.Machine \
+        io.systemd.Machine.List "{\"name\":\"$machine\"}" | jq -r '.leader.pid')"
+
+    for fd in /proc/"$pid"/fd/*; do
+        target="$(readlink "$fd" 2>/dev/null)" || continue
+        # The overlay is an O_TMPFILE. On file systems without O_TMPFILE, vmspawn creates a named
+        # temporary file instead and unlinks it right away.
+        if [[ "$target" == "$dir"/#*" (deleted)" || "$target" == "$dir"/systemd-tmp-*" (deleted)" ]]; then
+            echo "$fd"
+            return 0
+        fi
+    done
+
+    echo "No ephemeral overlay in $dir found among the fds of QEMU:" >&2
+    ls -l /proc/"$pid"/fd >&2
+    return 1
+}
+
 # Create a minimal root filesystem directory, then bake it into a raw ext4 image.
 # The guest doesn't need to fully boot — 'sleep infinity' keeps QEMU alive for QMP testing.
 mkdir -p "$WORKDIR/rootfs/sbin"
@@ -144,13 +169,18 @@ echo "Multi-drive VM terminated cleanly"
 # background job, JOB_STATUS_CHANGE events are watched, and when the job
 # concludes the deferred continuation fires blockdev-add (overlay format) +
 # device_add. If any step fails, the root drive is never attached and the kernel
-# panics — vmspawn exits without registering.
+# panics — vmspawn exits without registering. The image is in a read-only
+# directory, so the overlay has to be created in $TMPDIR instead.
+
+mkdir "$WORKDIR/ro" "$WORKDIR/large-tmp"
+ln "$WORKDIR/root.raw" "$WORKDIR/ro/root.raw"
+mount --bind -o ro "$WORKDIR/ro" "$WORKDIR/ro"
 
 MACHINE_EPHEMERAL="test-vmspawn-ephemeral-$$"
-systemd-vmspawn \
+TMPDIR="$WORKDIR/large-tmp" systemd-vmspawn \
     --machine="$MACHINE_EPHEMERAL" \
     --ram=256M \
-    --image="$WORKDIR/root.raw" \
+    --image="$WORKDIR/ro/root.raw" \
     --ephemeral \
     --linux="$KERNEL" \
     --tpm=no \
@@ -177,6 +207,9 @@ if grep -E '(add-fd|blockdev-add|blockdev-create|device_add|getfd|netdev_add|cha
 fi
 echo "No QMP device setup errors in ephemeral log"
 
+OVERLAY="$(find_overlay_fd "$MACHINE_EPHEMERAL" "$WORKDIR/large-tmp")"
+echo "Ephemeral overlay was created in \$TMPDIR and is QEMU fd ${OVERLAY##*/}"
+
 machinectl terminate "$MACHINE_EPHEMERAL"
 timeout 10 bash -c "while machinectl status '$MACHINE_EPHEMERAL' &>/dev/null; do sleep .5; done"
 timeout 10 bash -c "while kill -0 '$VMSPAWN_EPHEMERAL_PID' 2>/dev/null; do sleep .5; done"
@@ -185,7 +218,11 @@ echo "Ephemeral VM terminated cleanly"
 # --- Test 3: Ephemeral overlay grown with --grow-image= ---
 # In ephemeral mode the requested size has to land on the qcow2 overlay. The
 # image passed to --image= is opened read-only and has to come out of the run
-# at its original size.
+# at its original size. The image is passed through a symlink in another
+# directory, and the overlay has to be created next to the symlink target.
+
+mkdir "$WORKDIR/links"
+ln -s "$WORKDIR/root.raw" "$WORKDIR/links/root.raw"
 
 MACHINE_GROW="test-vmspawn-grow-$$"
 GROW_SIZE=$((512 * 1024 * 1024))
@@ -194,7 +231,7 @@ IMAGE_SIZE="$(stat -c %s "$WORKDIR/root.raw")"
 systemd-vmspawn \
     --machine="$MACHINE_GROW" \
     --ram=256M \
-    --image="$WORKDIR/root.raw" \
+    --image="$WORKDIR/links/root.raw" \
     --ephemeral \
     --grow-image="$GROW_SIZE" \
     --linux="$KERNEL" \
@@ -210,21 +247,8 @@ echo "Grown ephemeral machine '$MACHINE_GROW' registered with machined"
 assert_eq "$(stat -c %s "$WORKDIR/root.raw")" "$IMAGE_SIZE"
 echo "Image passed to --image= was left at its original size"
 
-# The overlay is unlinked, so QEMU's fd is the only way to reach it. vmspawn
-# registers QEMU itself as the machine leader, so machined knows its PID.
-QEMU_PID="$(varlinkctl call /run/systemd/machine/io.systemd.Machine \
-    io.systemd.Machine.List "{\"name\":\"$MACHINE_GROW\"}" | jq -r '.leader.pid')"
-
-OVERLAY=""
-for fd in /proc/"$QEMU_PID"/fd/*; do
-    target="$(readlink "$fd" 2>/dev/null)" || continue
-    # O_TMPFILE in the runtime directory, or the memfd fallback.
-    [[ "$target" == */vmspawn/"$MACHINE_GROW"/#* || "$target" == /memfd:vmspawn-overlay* ]] || continue
-    OVERLAY="$fd"
-    break
-done
-assert_neq "$OVERLAY" ""
-echo "Ephemeral overlay is QEMU fd ${OVERLAY##*/}"
+OVERLAY="$(find_overlay_fd "$MACHINE_GROW" "$WORKDIR")"
+echo "Ephemeral overlay was created next to the image and is QEMU fd ${OVERLAY##*/}"
 
 # qcow2 header: the magic, followed by the virtual size as a big endian u64 at
 # offset 24. Read it out of the header directly, qemu-img may not be installed.