udev-watch: reimplement with fanotify

Replace the inotify + /run/udev/watch/ symlink machinery with fanotify inode
marks (FAN_CLOSE_WRITE). The point is not merely a different backend; fanotify
removes several structural costs that inotify imposed on udevd:

- An inotify event only identifies the watch by a watch descriptor, which udevd
  had to translate back to a device. That mapping was maintained as a table of
  symlinks under /run/udev/watch/. A fanotify event instead carries the watched
  inode's file handle (FAN_REPORT_FID), which is resolved to the device directly
  via open_by_handle_at()+fstat(). The wd->device translation, and with it
  /run/udev/watch/ entirely, is no longer needed.

- Without that shared symlink store to coordinate, a worker can add and remove
  its own device marks directly on the fanotify group fd. Previously the watch
  state was owned by the manager, and workers had to request (un)registration
  over the notification socket (sd_notify()+SIGUSR1). Dropping that round-trip
  both simplifies the code and is expected to be faster.

- A fanotify mark lives on the inode and is dropped by the kernel once the inode
  is gone, so the special handling for removed device nodes is no longer needed.

The group fd is handed over across restarts via the service manager's fd store
and the kernel keeps the marks, so nothing is rebuilt in-process; the fd is also
propagated to each worker. Watches left behind by an older, inotify-based udevd
under /run/udev/watch/ are migrated to fanotify marks on first start, then the
directory is removed.

Device watch falls back to inotify on kernels < v5.13, when CONFIG_SHMEM=n
(devtmpfs backed by ramfs), or when CONFIG_FANOTIFY is disabled. This fallback
can also be forced via $SYSTEMD_UDEV_USE_INOTIFY.

docs/ENVIRONMENT.md

@@ -287,6 +287,9 @@ All tools:
   the worker process is killed by the manager process. Defaults to 10 seconds,
   maximum allowed is 5 hours.
 
+* `$SYSTEMD_UDEV_USE_INOTIFY=` — If true, use inotify instead of fanotify for
+  device node watches (`OPTIONS+="watch"`). Defaults to false.
+
 `udevadm` and `systemd-hwdb`:
 
 * `SYSTEMD_HWDB_UPDATE_BYPASS=` — If set to "1", execution of hwdb updates is skipped

src/udev/meson.build

@@ -42,7 +42,8 @@ udevadm_export_sources = files(
         'udev-spawn.c',
         'udev-synth.c',
         'udev-varlink.c',
-        'udev-watch.c',
+        'udev-watch-fanotify.c',
+        'udev-watch-inotify.c',
         'udev-worker.c',
 )
 

src/udev/udev-manager.c

@@ -167,6 +167,8 @@ Manager* manager_free(Manager *manager) {
         /* udev watch */
         sd_event_source_unref(manager->device_watch_event);
         safe_close(manager->inotify_fd);
+        safe_close(manager->fanotify_fd);
+        safe_close(manager->dev_fd);
 
         /* udev synth */
         set_free(manager->synthesize_change_child_event_sources);
@@ -195,6 +197,8 @@ Manager* manager_new(void) {
 
         *manager = (Manager) {
                 .inotify_fd = -EBADF,
+                .fanotify_fd = -EBADF,
+                .dev_fd = -EBADF,
                 .config_by_udev_conf = UDEV_CONFIG_INIT,
                 .config_by_command = UDEV_CONFIG_INIT,
                 .config_by_kernel = UDEV_CONFIG_INIT,
@@ -549,6 +553,7 @@ static int worker_spawn(Manager *manager, Event *event) {
                         .rules = TAKE_PTR(manager->rules),
                         .config = manager->config,
                         .manager_pid = manager_pid,
+                        .fanotify_fd = manager->fanotify_fd,
                 };
 
                 if (manager->workers_cgroup) {
@@ -1295,7 +1300,8 @@ static int on_post_exit(Manager *manager) {
 
         (void) manager_serialize_events(manager);
 
-        udev_watch_dump();
+        udev_watch_dump(manager);
+
         return sd_event_exit(manager->event, 0);
 }
 
@@ -1449,6 +1455,8 @@ static int manager_listen_fds(Manager *manager, int *ret_varlink_fd) {
                         r = manager_init_device_monitor(manager, fd);
                 else if (streq(names[i], "inotify"))
                         r = manager_init_inotify(manager, fd);
+                else if (streq(names[i], "fanotify"))
+                        r = manager_init_fanotify(manager, fd);
                 else if (streq(names[i], "config-serialization"))
                         r = manager_deserialize_config(manager, &fd);
                 else if (streq(names[i], "event-serialization"))
@@ -1514,7 +1522,7 @@ int manager_main(Manager *manager) {
         if (r < 0)
                 return r;
 
-        r = manager_start_inotify(manager);
+        r = manager_start_device_watch(manager);
         if (r < 0)
                 return r;
 

src/udev/udev-manager.h

@@ -1,6 +1,8 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 #pragma once
 
+#include <sys/statfs.h>
+
 #include "sd-device.h"
 #include "sd-event.h"
 
@@ -56,6 +58,9 @@ typedef struct Manager {
 
         /* used by udev-watch */
         int inotify_fd;
+        int fanotify_fd;
+        int dev_fd;
+        struct statfs dev_statfs;
         sd_event_source *device_watch_event;
 
         /* used by udev-synth */

src/udev/udev-watch-fanotify.c

@@ -0,0 +1,287 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright © 2009 Canonical Ltd.
+ * Copyright © 2009 Scott James Remnant <scott@netsplit.com>
+ */
+
+#include <sys/fanotify.h>
+#include <unistd.h>
+
+#include "daemon-util.h"
+#include "device-util.h"
+#include "dirent-util.h"
+#include "errno-util.h"
+#include "fanotify-util.h"
+#include "fd-util.h"
+#include "rm-rf.h"
+#include "string-util.h"
+#include "udev-manager.h"
+#include "udev-synth.h"
+#include "udev-util.h"
+#include "udev-watch.h"
+#include "udev-worker.h"
+
+static int device_watch(int fanotify_fd, sd_device *dev, bool add) {
+        assert(fanotify_fd >= 0);
+        assert(dev);
+
+        _cleanup_close_ int fd = sd_device_open(dev, O_CLOEXEC | O_NOFOLLOW | O_PATH);
+        if (fd < 0)
+                return fd;
+
+        return fanotify_mark_fd(
+                        fanotify_fd,
+                        (add ? FAN_MARK_ADD : FAN_MARK_REMOVE) | FAN_MARK_INODE,
+                        FAN_CLOSE_WRITE,
+                        fd);
+}
+
+void udev_watch_begin(UdevWorker *worker, sd_device *dev) {
+        int r;
+
+        assert(worker);
+        assert(dev);
+
+        /* Ignore the request of watching the device node on remove event, as the device node specified by
+         * DEVNAME= has already been removed, and may already be assigned to another device. Consider the
+         * case e.g. a USB stick memory was unplugged and then another one is plugged. */
+        if (device_for_action(dev, SD_DEVICE_REMOVE)) {
+                log_device_debug(dev, "Not adding device watch on remove uevent.");
+                return;
+        }
+
+        if (worker->fanotify_fd < 0)
+                return inotify_watch_begin(worker, dev);
+
+        r = device_watch(worker->fanotify_fd, dev, /* add= */ true);
+        if (r == -ENOEXEC)
+                return; /* The device does not have device node. */
+        if (r < 0)
+                log_device_warning_errno(dev, r, "Failed to add device watch, ignoring: %m");
+        else
+                log_device_debug(dev, "Added device watch.");
+}
+
+void udev_watch_end(UdevWorker *worker, sd_device *dev) {
+        int r;
+
+        assert(worker);
+        assert(dev);
+
+        if (worker->fanotify_fd < 0)
+                return inotify_watch_end(worker, dev);
+
+        r = device_watch(worker->fanotify_fd, dev, /* add= */ false);
+        if (IN_SET(r, -ENOEXEC, -ENOENT))
+                /* ENOEXEC : The device does not have device node. */
+                /* ENOENT  : The node is not watched, or the device node is already gone. */
+                return;
+        if (r < 0)
+                log_device_warning_errno(dev, r, "Failed to remove device watch, ignoring: %m");
+        else
+                log_device_debug(dev, "Removed device watch.");
+}
+
+static int manager_restore_device_watch(Manager *manager) {
+        _cleanup_(rm_rf_safep) const char *old = "/run/udev/watch.old/";
+        int r;
+
+        /* Migrate watches registered by an older, inotify-based udevd, which stored them as symlinks under
+         * /run/udev/watch/. Re-add them as fanotify marks, then drop the directory: the fanotify
+         * implementation does not use it. Downgrading back to inotify is not supported; after a downgrade,
+         * re-enabling the watches requires running 'udevadm trigger' for all devices that have a device node. */
+
+        assert(manager);
+        assert(manager->fanotify_fd >= 0);
+
+        rm_rf_safe(old);
+        if (rename("/run/udev/watch/", old) < 0) {
+                if (errno == ENOENT)
+                        return 0;
+
+                return log_warning_errno(errno, "Failed to move watches directory '/run/udev/watch/': %m");
+        }
+
+        _cleanup_closedir_ DIR *dir = opendir(old);
+        if (!dir)
+                return log_warning_errno(errno, "Failed to open old watches directory '%s': %m", old);
+
+        FOREACH_DIRENT(de, dir, break) {
+                if (in_charset(de->d_name, DIGITS))
+                        continue; /* This should be wd -> ID symlink. Skipping. */
+
+                _cleanup_(sd_device_unrefp) sd_device *dev = NULL;
+                r = sd_device_new_from_device_id(&dev, de->d_name);
+                if (r < 0) {
+                        log_full_errno(ERRNO_IS_NEG_DEVICE_ABSENT(r) ? LOG_DEBUG : LOG_WARNING, r,
+                                       "Failed to create sd_device object from device ID '%s', ignoring: %m",
+                                       de->d_name);
+                        continue;
+                }
+
+                r = device_watch(manager->fanotify_fd, dev, /* add= */ true);
+                if (r < 0)
+                        log_device_debug_errno(dev, r, "Failed to add device watch, ignoring: %m");
+        }
+
+        return 0;
+}
+
+static int manager_process_fanotify(Manager *manager, const struct fanotify_event_metadata *e) {
+        int r;
+
+        assert(manager);
+        assert(e);
+
+        if (!FLAGS_SET(e->mask, FAN_CLOSE_WRITE))
+                return 0;
+
+        _cleanup_close_ int fd = fanotify_event_pin_fd(e, manager->dev_fd, &manager->dev_statfs);
+        if (fd < 0)
+                return log_debug_errno(fd, "Failed to get file descriptor from fanotify event, ignoring: %m");
+
+        _cleanup_(sd_device_unrefp) sd_device *dev = NULL;
+        r = sd_device_new_from_device_node_fd(&dev, fd);
+        if (r < 0) /* Device may be removed just after closed. */
+                return log_debug_errno(r, "Failed to resolve device from file descriptor obtained by fanotify event, ignoring: %m");
+
+        log_device_debug(dev, "Received fanotify close-after-write event.");
+
+        (void) manager_create_queue_file(manager);
+        (void) manager_requeue_locked_events_by_device(manager, dev);
+        (void) manager_synthesize_change(manager, dev);
+        return 0;
+}
+
+static int on_fanotify(sd_event_source *s, int fd, uint32_t revents, void *userdata) {
+        Manager *manager = ASSERT_PTR(userdata);
+
+        assert(fd >= 0);
+
+        union fanotify_event_buffer buffer;
+        ssize_t l = read(fd, &buffer, sizeof(buffer));
+        if (l < 0) {
+                if (ERRNO_IS_TRANSIENT(errno))
+                        return 0;
+
+                return log_error_errno(errno, "Failed to read fanotify events: %m");
+        }
+
+        bool overflow = false;
+        FOREACH_FANOTIFY_EVENT(e, buffer, l) {
+                if (e->vers != FANOTIFY_METADATA_VERSION)
+                        return log_error_errno(SYNTHETIC_ERRNO(EPROTO),
+                                               "fanotify metadata version mismatch (got %u, expected %u).",
+                                               e->vers, (unsigned) FANOTIFY_METADATA_VERSION);
+
+                (void) manager_process_fanotify(manager, e);
+
+                if (FLAGS_SET(e->mask, FAN_Q_OVERFLOW))
+                        overflow = true;
+        }
+        if (overflow)
+                log_debug("fanotify event queue overflow detected, some close events might be dropped.");
+
+        return 0;
+}
+
+int manager_init_fanotify(Manager *manager, int fd) {
+        int r;
+
+        assert(manager);
+
+        /* This takes passed file descriptor on success. */
+
+        r = secure_getenv_bool("SYSTEMD_UDEV_USE_INOTIFY");
+        if (r < 0 && r != -ENXIO)
+                log_debug_errno(r, "Failed to parse $SYSTEMD_UDEV_USE_INOTIFY, ignoring: %m");
+        if (r > 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(EOPNOTSUPP),
+                                       "Using inotify for device watch (as requested by $SYSTEMD_UDEV_USE_INOTIFY).");
+
+        if (manager->dev_fd < 0) {
+                _cleanup_close_ int dev_fd = open("/dev/", O_CLOEXEC | O_DIRECTORY);
+                if (dev_fd < 0)
+                        return log_error_errno(errno, "Failed to open '/dev/': %m");
+
+                if (fstatfs(dev_fd, &manager->dev_statfs) < 0)
+                        return log_error_errno(errno, "Failed to statfs '/dev/': %m");
+
+                /* fanotify can only watch device nodes via FAN_REPORT_FID (file handle mode). FAN_REPORT_FID
+                 * requires both non-zero f_fsid and file handle support; devtmpfs provides these only when
+                 * backed by tmpfs on kernels ≥ v5.13 (see commit 59cda49ecf6c9a32fae4942420701b6e087204f6).
+                 * On older kernels, or when CONFIG_SHMEM=n (making devtmpfs use ramfs instead), device watch
+                 * falls back to inotify. */
+                if (memeqzero(&manager->dev_statfs.f_fsid, sizeof(manager->dev_statfs.f_fsid)))
+                        return log_debug_errno(SYNTHETIC_ERRNO(EOPNOTSUPP),
+                                               "The kernel does not support fanotify for device nodes, falling back to use inotify.");
+
+                manager->dev_fd = TAKE_FD(dev_fd);
+        }
+
+        if (fd >= 0) {
+                if (manager->fanotify_fd >= 0)
+                        return log_warning_errno(SYNTHETIC_ERRNO(EALREADY), "Received multiple fanotify fd (%i), ignoring.", fd);
+
+                log_debug("Received fanotify fd (%i) from service manager.", fd);
+                manager->fanotify_fd = fd;
+                return 0;
+        }
+
+        if (manager->fanotify_fd >= 0)
+                return 0;
+
+        fd = fanotify_init(
+                        FAN_CLASS_NOTIF | FAN_CLOEXEC | FAN_NONBLOCK | FAN_REPORT_FID,
+                        O_CLOEXEC | O_NONBLOCK | O_RDONLY);
+        if (fd < 0) {
+                /* CONFIG_FANOTIFY is disabled. */
+                if (ERRNO_IS_NOT_SUPPORTED(errno))
+                        return log_debug_errno(errno, "Failed to create fanotify group, falling back to use inotify: %m");
+
+                return log_error_errno(errno, "Failed to create fanotify group: %m");
+        }
+
+        log_debug("Initialized new fanotify group.");
+        manager->fanotify_fd = fd;
+
+        (void) manager_restore_device_watch(manager);
+
+        r = notify_push_fd(manager->fanotify_fd, "fanotify");
+        if (r < 0)
+                log_warning_errno(r, "Failed to push fanotify fd to service manager, ignoring: %m");
+        else
+                log_debug("Pushed fanotify fd to service manager.");
+
+        return 0;
+}
+
+int manager_start_device_watch(Manager *manager) {
+        int r;
+
+        assert(manager);
+        assert(manager->event);
+
+        r = manager_init_fanotify(manager, -EBADF);
+        if (ERRNO_IS_NEG_NOT_SUPPORTED(r))
+                return manager_start_inotify(manager);
+        if (r < 0)
+                return r;
+
+        /* Drop unused inotify fd from fdstore. */
+        manager->inotify_fd = close_and_notify_warn(manager->inotify_fd, "inotify");
+
+        _cleanup_(sd_event_source_unrefp) sd_event_source *s = NULL;
+        r = sd_event_add_io(manager->event, &s, manager->fanotify_fd, EPOLLIN, on_fanotify, manager);
+        if (r < 0)
+                return log_error_errno(r, "Failed to create event source for device watch: %m");
+
+        r = sd_event_source_set_priority(s, EVENT_PRIORITY_DEVICE_WATCH);
+        if (r < 0)
+                return log_error_errno(r, "Failed to set priority to event source for device watch: %m");
+
+        (void) sd_event_source_set_description(s, "manager-device-watch");
+
+        manager->device_watch_event = TAKE_PTR(s);
+        return 0;
+}

src/udev/udev-watch-inotify.c

@@ -52,9 +52,15 @@ static int device_new_from_watch_handle_at(sd_device **ret, int dirfd, int wd) {
         return sd_device_new_from_device_id(ret, id);
 }
 
-void udev_watch_dump(void) {
+void udev_watch_dump(Manager *manager) {
         int r;
 
+        assert(manager);
+
+        /* This is inotify specific. Do nothing if inotify is disabled. */
+        if (manager->inotify_fd < 0)
+                return;
+
         _cleanup_closedir_ DIR *dir = opendir("/run/udev/watch/");
         if (!dir)
                 return (void) log_full_errno(errno == ENOENT ? LOG_DEBUG : LOG_WARNING, errno,
@@ -276,7 +282,7 @@ int manager_start_inotify(Manager *manager) {
         if (r < 0)
                 return r;
 
-        udev_watch_dump();
+        udev_watch_dump(manager);
 
         _cleanup_(sd_event_source_unrefp) sd_event_source *s = NULL;
         r = sd_event_add_io(manager->event, &s, manager->inotify_fd, EPOLLIN, on_inotify, manager);
@@ -533,20 +539,12 @@ static int notify_and_wait_signal(UdevWorker *worker, sd_device *dev, const char
         return sd_event_loop(e);
 }
 
-void udev_watch_begin(UdevWorker *worker, sd_device *dev) {
+void inotify_watch_begin(UdevWorker *worker, sd_device *dev) {
         int r;
 
         assert(worker);
         assert(dev);
 
-        /* Ignore the request of watching the device node on remove event, as the device node specified by
-         * DEVNAME= has already been removed, and may already be assigned to another device. Consider the
-         * case e.g. a USB stick memory was unplugged and then another one is plugged. */
-        if (device_for_action(dev, SD_DEVICE_REMOVE)) {
-                log_device_debug(dev, "Not adding device watch on remove uevent.");
-                return;
-        }
-
         r = notify_and_wait_signal(worker, dev, "INOTIFY_WATCH_ADD=1");
         if (r < 0)
                 log_device_warning_errno(dev, r, "Failed to add device watch, ignoring: %m");
@@ -554,7 +552,7 @@ void udev_watch_begin(UdevWorker *worker, sd_device *dev) {
                 log_device_debug(dev, "Added device watch.");
 }
 
-void udev_watch_end(UdevWorker *worker, sd_device *dev) {
+void inotify_watch_end(UdevWorker *worker, sd_device *dev) {
         int r;
 
         assert(worker);

src/udev/udev-watch.h

@@ -3,13 +3,18 @@
 
 #include "udev-forward.h"
 
-void udev_watch_dump(void);
-
+/* inotify backend */
+// FIXME: drop support for inotify once fanotify is in our baseline, i.e. 5.13
+void udev_watch_dump(Manager *manager);
 int manager_init_inotify(Manager *manager, int fd);
 int manager_start_inotify(Manager *manager);
-
 int manager_add_watch(Manager *manager, sd_device *dev);
 int manager_remove_watch(Manager *manager, sd_device *dev);
+void inotify_watch_begin(UdevWorker *worker, sd_device *dev);
+void inotify_watch_end(UdevWorker *worker, sd_device *dev);
 
+/* fanotify backend */
 void udev_watch_begin(UdevWorker *worker, sd_device *dev);
 void udev_watch_end(UdevWorker *worker, sd_device *dev);
+int manager_init_fanotify(Manager *manager, int fd);
+int manager_start_device_watch(Manager *manager);

src/udev/udev-worker.h

@@ -17,7 +17,8 @@ typedef struct UdevWorker {
 
         UdevConfig config;
 
-        pid_t manager_pid;
+        pid_t manager_pid; /* used by inotify backed device watch */
+        int fanotify_fd;
 } UdevWorker;
 
 void udev_worker_done(UdevWorker *worker);

test/units/TEST-17-UDEV.watch-fanotify.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: LGPL-2.1-or-later
+set -ex
+set -o pipefail
+
+# tests for udev watch
+
+# shellcheck source=test/units/util.sh
+. "$(dirname "$0")"/util.sh
+
+udevd_fanotify_fdinfo() {
+    local pid fd
+
+    # Print the path of systemd-udevd's fanotify group fdinfo.
+
+    pid="$(systemctl show --property MainPID --value systemd-udevd.service)"
+    [[ "${pid:-0}" -gt 0 ]] || return 1
+
+    for fd in /proc/"$pid"/fd/*; do
+        if [[ "$(readlink "$fd" 2>/dev/null)" == "anon_inode:[fanotify]" ]]; then
+            echo "/proc/$pid/fdinfo/${fd##*/}"
+            return 0
+        fi
+    done
+
+    return 1
+}
+
+device_is_watched() {
+    local devnode="${1:?}"
+    local ino fdinfo
+
+    fdinfo="$(udevd_fanotify_fdinfo)" || exit 1
+    ino="$(printf '%x' "$(stat -c '%i' "$devnode")")"
+    grep -qE "^fanotify ino:${ino} " "$fdinfo"
+}
+
+ROOTDEV="$(bootctl -RR)"
+ROOTDEV_NAME="$(udevadm info --query=name "$ROOTDEV")"
+
+mkdir -p /run/udev/rules.d/
+
+cat >/run/udev/rules.d/99-testsuite.rules <<EOF
+ACTION=="change", SUBSYSTEM=="block", KERNEL=="${ROOTDEV_NAME}", OPTIONS:="nowatch"
+EOF
+
+udevadm control --reload
+udevadm trigger --action change --settle "$ROOTDEV"
+(! device_is_watched "$ROOTDEV")
+
+cat >/run/udev/rules.d/99-testsuite.rules <<EOF
+ACTION=="change", SUBSYSTEM=="block", KERNEL=="${ROOTDEV_NAME}", OPTIONS:="watch"
+EOF
+
+udevadm control --reload
+udevadm trigger --action change --settle "$ROOTDEV"
+device_is_watched "$ROOTDEV"
+
+rm /run/udev/rules.d/99-testsuite.rules
+udevadm control --reload
+
+exit 0

test/units/TEST-17-UDEV.watch-inotify.sh

@@ -91,9 +91,13 @@ mkdir -p /run/systemd/system/systemd-udevd.service.d/
 cat >/run/systemd/system/systemd-udevd.service.d/10-debug.conf <<EOF
 [Service]
 Environment=SYSTEMD_LOG_LEVEL=debug
+Environment=SYSTEMD_UDEV_USE_INOTIFY=1
 EOF
 
 systemctl daemon-reload
+systemctl restart systemd-udevd.service
+
+udevadm trigger -w --action change --subsystem-match=block
 
 mkdir -p /run/udev/rules.d/
 

units/systemd-udevd.service.in

@@ -53,7 +53,7 @@ MemoryDenyWriteExecute=yes
 RestrictAddressFamilies=AF_UNIX AF_NETLINK AF_INET AF_INET6
 RestrictRealtime=yes
 RestrictSUIDSGID=yes
-SystemCallFilter=@system-service @module @raw-io bpf
+SystemCallFilter=@system-service @module @raw-io bpf fanotify_init fanotify_mark open_by_handle_at
 SystemCallFilter=~@clock
 SystemCallErrorNumber=EPERM
 SystemCallArchitectures=native