From 6d8af5c208c746b3746cc3696dfe9eb33508e1f4 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Thu, 8 Oct 2026 10:14:18 +0900 Subject: [PATCH] udev-watch: reimplement with fanotify Replace the inotify + /run/udev/watch/ symlink machinery with fanotify inode marks (FAN_CLOSE_WRITE). The point is not merely a different backend; fanotify removes several structural costs that inotify imposed on udevd: - An inotify event only identifies the watch by a watch descriptor, which udevd had to translate back to a device. That mapping was maintained as a table of symlinks under /run/udev/watch/. A fanotify event instead carries the watched inode's file handle (FAN_REPORT_FID), which is resolved to the device directly via open_by_handle_at()+fstat(). The wd->device translation, and with it /run/udev/watch/ entirely, is no longer needed. - Without that shared symlink store to coordinate, a worker can add and remove its own device marks directly on the fanotify group fd. Previously the watch state was owned by the manager, and workers had to request (un)registration over the notification socket (sd_notify()+SIGUSR1). Dropping that round-trip both simplifies the code and is expected to be faster. - A fanotify mark lives on the inode and is dropped by the kernel once the inode is gone, so the special handling for removed device nodes is no longer needed. The group fd is handed over across restarts via the service manager's fd store and the kernel keeps the marks, so nothing is rebuilt in-process; the fd is also propagated to each worker. Watches left behind by an older, inotify-based udevd under /run/udev/watch/ are migrated to fanotify marks on first start, then the directory is removed. Device watch falls back to inotify on kernels < v5.13, when CONFIG_SHMEM=n (devtmpfs backed by ramfs), or when CONFIG_FANOTIFY is disabled. This fallback can also be forced via $SYSTEMD_UDEV_USE_INOTIFY. --- docs/ENVIRONMENT.md | 3 + src/udev/meson.build | 3 +- src/udev/udev-manager.c | 12 +- src/udev/udev-manager.h | 5 + src/udev/udev-watch-fanotify.c | 287 ++++++++++++++++++ .../{udev-watch.c => udev-watch-inotify.c} | 22 +- src/udev/udev-watch.h | 11 +- src/udev/udev-worker.h | 3 +- test/units/TEST-17-UDEV.watch-fanotify.sh | 62 ++++ ...watch.sh => TEST-17-UDEV.watch-inotify.sh} | 4 + units/systemd-udevd.service.in | 2 +- 11 files changed, 394 insertions(+), 20 deletions(-) create mode 100644 src/udev/udev-watch-fanotify.c rename src/udev/{udev-watch.c => udev-watch-inotify.c} (96%) create mode 100755 test/units/TEST-17-UDEV.watch-fanotify.sh rename test/units/{TEST-17-UDEV.watch.sh => TEST-17-UDEV.watch-inotify.sh} (96%) diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index dbaa2976f50..d9453b5d3ce 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -287,6 +287,9 @@ All tools: the worker process is killed by the manager process. Defaults to 10 seconds, maximum allowed is 5 hours. +* `$SYSTEMD_UDEV_USE_INOTIFY=` — If true, use inotify instead of fanotify for + device node watches (`OPTIONS+="watch"`). Defaults to false. + `udevadm` and `systemd-hwdb`: * `SYSTEMD_HWDB_UPDATE_BYPASS=` — If set to "1", execution of hwdb updates is skipped diff --git a/src/udev/meson.build b/src/udev/meson.build index 1a76e625cc6..8cfd503a69c 100644 --- a/src/udev/meson.build +++ b/src/udev/meson.build @@ -42,7 +42,8 @@ udevadm_export_sources = files( 'udev-spawn.c', 'udev-synth.c', 'udev-varlink.c', - 'udev-watch.c', + 'udev-watch-fanotify.c', + 'udev-watch-inotify.c', 'udev-worker.c', ) diff --git a/src/udev/udev-manager.c b/src/udev/udev-manager.c index de5f894c616..107a5ce060c 100644 --- a/src/udev/udev-manager.c +++ b/src/udev/udev-manager.c @@ -167,6 +167,8 @@ Manager* manager_free(Manager *manager) { /* udev watch */ sd_event_source_unref(manager->device_watch_event); safe_close(manager->inotify_fd); + safe_close(manager->fanotify_fd); + safe_close(manager->dev_fd); /* udev synth */ set_free(manager->synthesize_change_child_event_sources); @@ -195,6 +197,8 @@ Manager* manager_new(void) { *manager = (Manager) { .inotify_fd = -EBADF, + .fanotify_fd = -EBADF, + .dev_fd = -EBADF, .config_by_udev_conf = UDEV_CONFIG_INIT, .config_by_command = UDEV_CONFIG_INIT, .config_by_kernel = UDEV_CONFIG_INIT, @@ -549,6 +553,7 @@ static int worker_spawn(Manager *manager, Event *event) { .rules = TAKE_PTR(manager->rules), .config = manager->config, .manager_pid = manager_pid, + .fanotify_fd = manager->fanotify_fd, }; if (manager->workers_cgroup) { @@ -1295,7 +1300,8 @@ static int on_post_exit(Manager *manager) { (void) manager_serialize_events(manager); - udev_watch_dump(); + udev_watch_dump(manager); + return sd_event_exit(manager->event, 0); } @@ -1449,6 +1455,8 @@ static int manager_listen_fds(Manager *manager, int *ret_varlink_fd) { r = manager_init_device_monitor(manager, fd); else if (streq(names[i], "inotify")) r = manager_init_inotify(manager, fd); + else if (streq(names[i], "fanotify")) + r = manager_init_fanotify(manager, fd); else if (streq(names[i], "config-serialization")) r = manager_deserialize_config(manager, &fd); else if (streq(names[i], "event-serialization")) @@ -1514,7 +1522,7 @@ int manager_main(Manager *manager) { if (r < 0) return r; - r = manager_start_inotify(manager); + r = manager_start_device_watch(manager); if (r < 0) return r; diff --git a/src/udev/udev-manager.h b/src/udev/udev-manager.h index 44a3d01013b..26c7c236438 100644 --- a/src/udev/udev-manager.h +++ b/src/udev/udev-manager.h @@ -1,6 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ #pragma once +#include + #include "sd-device.h" #include "sd-event.h" @@ -56,6 +58,9 @@ typedef struct Manager { /* used by udev-watch */ int inotify_fd; + int fanotify_fd; + int dev_fd; + struct statfs dev_statfs; sd_event_source *device_watch_event; /* used by udev-synth */ diff --git a/src/udev/udev-watch-fanotify.c b/src/udev/udev-watch-fanotify.c new file mode 100644 index 00000000000..39607f9cee9 --- /dev/null +++ b/src/udev/udev-watch-fanotify.c @@ -0,0 +1,287 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright © 2009 Canonical Ltd. + * Copyright © 2009 Scott James Remnant + */ + +#include +#include + +#include "daemon-util.h" +#include "device-util.h" +#include "dirent-util.h" +#include "errno-util.h" +#include "fanotify-util.h" +#include "fd-util.h" +#include "rm-rf.h" +#include "string-util.h" +#include "udev-manager.h" +#include "udev-synth.h" +#include "udev-util.h" +#include "udev-watch.h" +#include "udev-worker.h" + +static int device_watch(int fanotify_fd, sd_device *dev, bool add) { + assert(fanotify_fd >= 0); + assert(dev); + + _cleanup_close_ int fd = sd_device_open(dev, O_CLOEXEC | O_NOFOLLOW | O_PATH); + if (fd < 0) + return fd; + + return fanotify_mark_fd( + fanotify_fd, + (add ? FAN_MARK_ADD : FAN_MARK_REMOVE) | FAN_MARK_INODE, + FAN_CLOSE_WRITE, + fd); +} + +void udev_watch_begin(UdevWorker *worker, sd_device *dev) { + int r; + + assert(worker); + assert(dev); + + /* Ignore the request of watching the device node on remove event, as the device node specified by + * DEVNAME= has already been removed, and may already be assigned to another device. Consider the + * case e.g. a USB stick memory was unplugged and then another one is plugged. */ + if (device_for_action(dev, SD_DEVICE_REMOVE)) { + log_device_debug(dev, "Not adding device watch on remove uevent."); + return; + } + + if (worker->fanotify_fd < 0) + return inotify_watch_begin(worker, dev); + + r = device_watch(worker->fanotify_fd, dev, /* add= */ true); + if (r == -ENOEXEC) + return; /* The device does not have device node. */ + if (r < 0) + log_device_warning_errno(dev, r, "Failed to add device watch, ignoring: %m"); + else + log_device_debug(dev, "Added device watch."); +} + +void udev_watch_end(UdevWorker *worker, sd_device *dev) { + int r; + + assert(worker); + assert(dev); + + if (worker->fanotify_fd < 0) + return inotify_watch_end(worker, dev); + + r = device_watch(worker->fanotify_fd, dev, /* add= */ false); + if (IN_SET(r, -ENOEXEC, -ENOENT)) + /* ENOEXEC : The device does not have device node. */ + /* ENOENT : The node is not watched, or the device node is already gone. */ + return; + if (r < 0) + log_device_warning_errno(dev, r, "Failed to remove device watch, ignoring: %m"); + else + log_device_debug(dev, "Removed device watch."); +} + +static int manager_restore_device_watch(Manager *manager) { + _cleanup_(rm_rf_safep) const char *old = "/run/udev/watch.old/"; + int r; + + /* Migrate watches registered by an older, inotify-based udevd, which stored them as symlinks under + * /run/udev/watch/. Re-add them as fanotify marks, then drop the directory: the fanotify + * implementation does not use it. Downgrading back to inotify is not supported; after a downgrade, + * re-enabling the watches requires running 'udevadm trigger' for all devices that have a device node. */ + + assert(manager); + assert(manager->fanotify_fd >= 0); + + rm_rf_safe(old); + if (rename("/run/udev/watch/", old) < 0) { + if (errno == ENOENT) + return 0; + + return log_warning_errno(errno, "Failed to move watches directory '/run/udev/watch/': %m"); + } + + _cleanup_closedir_ DIR *dir = opendir(old); + if (!dir) + return log_warning_errno(errno, "Failed to open old watches directory '%s': %m", old); + + FOREACH_DIRENT(de, dir, break) { + if (in_charset(de->d_name, DIGITS)) + continue; /* This should be wd -> ID symlink. Skipping. */ + + _cleanup_(sd_device_unrefp) sd_device *dev = NULL; + r = sd_device_new_from_device_id(&dev, de->d_name); + if (r < 0) { + log_full_errno(ERRNO_IS_NEG_DEVICE_ABSENT(r) ? LOG_DEBUG : LOG_WARNING, r, + "Failed to create sd_device object from device ID '%s', ignoring: %m", + de->d_name); + continue; + } + + r = device_watch(manager->fanotify_fd, dev, /* add= */ true); + if (r < 0) + log_device_debug_errno(dev, r, "Failed to add device watch, ignoring: %m"); + } + + return 0; +} + +static int manager_process_fanotify(Manager *manager, const struct fanotify_event_metadata *e) { + int r; + + assert(manager); + assert(e); + + if (!FLAGS_SET(e->mask, FAN_CLOSE_WRITE)) + return 0; + + _cleanup_close_ int fd = fanotify_event_pin_fd(e, manager->dev_fd, &manager->dev_statfs); + if (fd < 0) + return log_debug_errno(fd, "Failed to get file descriptor from fanotify event, ignoring: %m"); + + _cleanup_(sd_device_unrefp) sd_device *dev = NULL; + r = sd_device_new_from_device_node_fd(&dev, fd); + if (r < 0) /* Device may be removed just after closed. */ + return log_debug_errno(r, "Failed to resolve device from file descriptor obtained by fanotify event, ignoring: %m"); + + log_device_debug(dev, "Received fanotify close-after-write event."); + + (void) manager_create_queue_file(manager); + (void) manager_requeue_locked_events_by_device(manager, dev); + (void) manager_synthesize_change(manager, dev); + return 0; +} + +static int on_fanotify(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + Manager *manager = ASSERT_PTR(userdata); + + assert(fd >= 0); + + union fanotify_event_buffer buffer; + ssize_t l = read(fd, &buffer, sizeof(buffer)); + if (l < 0) { + if (ERRNO_IS_TRANSIENT(errno)) + return 0; + + return log_error_errno(errno, "Failed to read fanotify events: %m"); + } + + bool overflow = false; + FOREACH_FANOTIFY_EVENT(e, buffer, l) { + if (e->vers != FANOTIFY_METADATA_VERSION) + return log_error_errno(SYNTHETIC_ERRNO(EPROTO), + "fanotify metadata version mismatch (got %u, expected %u).", + e->vers, (unsigned) FANOTIFY_METADATA_VERSION); + + (void) manager_process_fanotify(manager, e); + + if (FLAGS_SET(e->mask, FAN_Q_OVERFLOW)) + overflow = true; + } + if (overflow) + log_debug("fanotify event queue overflow detected, some close events might be dropped."); + + return 0; +} + +int manager_init_fanotify(Manager *manager, int fd) { + int r; + + assert(manager); + + /* This takes passed file descriptor on success. */ + + r = secure_getenv_bool("SYSTEMD_UDEV_USE_INOTIFY"); + if (r < 0 && r != -ENXIO) + log_debug_errno(r, "Failed to parse $SYSTEMD_UDEV_USE_INOTIFY, ignoring: %m"); + if (r > 0) + return log_debug_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), + "Using inotify for device watch (as requested by $SYSTEMD_UDEV_USE_INOTIFY)."); + + if (manager->dev_fd < 0) { + _cleanup_close_ int dev_fd = open("/dev/", O_CLOEXEC | O_DIRECTORY); + if (dev_fd < 0) + return log_error_errno(errno, "Failed to open '/dev/': %m"); + + if (fstatfs(dev_fd, &manager->dev_statfs) < 0) + return log_error_errno(errno, "Failed to statfs '/dev/': %m"); + + /* fanotify can only watch device nodes via FAN_REPORT_FID (file handle mode). FAN_REPORT_FID + * requires both non-zero f_fsid and file handle support; devtmpfs provides these only when + * backed by tmpfs on kernels ≥ v5.13 (see commit 59cda49ecf6c9a32fae4942420701b6e087204f6). + * On older kernels, or when CONFIG_SHMEM=n (making devtmpfs use ramfs instead), device watch + * falls back to inotify. */ + if (memeqzero(&manager->dev_statfs.f_fsid, sizeof(manager->dev_statfs.f_fsid))) + return log_debug_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), + "The kernel does not support fanotify for device nodes, falling back to use inotify."); + + manager->dev_fd = TAKE_FD(dev_fd); + } + + if (fd >= 0) { + if (manager->fanotify_fd >= 0) + return log_warning_errno(SYNTHETIC_ERRNO(EALREADY), "Received multiple fanotify fd (%i), ignoring.", fd); + + log_debug("Received fanotify fd (%i) from service manager.", fd); + manager->fanotify_fd = fd; + return 0; + } + + if (manager->fanotify_fd >= 0) + return 0; + + fd = fanotify_init( + FAN_CLASS_NOTIF | FAN_CLOEXEC | FAN_NONBLOCK | FAN_REPORT_FID, + O_CLOEXEC | O_NONBLOCK | O_RDONLY); + if (fd < 0) { + /* CONFIG_FANOTIFY is disabled. */ + if (ERRNO_IS_NOT_SUPPORTED(errno)) + return log_debug_errno(errno, "Failed to create fanotify group, falling back to use inotify: %m"); + + return log_error_errno(errno, "Failed to create fanotify group: %m"); + } + + log_debug("Initialized new fanotify group."); + manager->fanotify_fd = fd; + + (void) manager_restore_device_watch(manager); + + r = notify_push_fd(manager->fanotify_fd, "fanotify"); + if (r < 0) + log_warning_errno(r, "Failed to push fanotify fd to service manager, ignoring: %m"); + else + log_debug("Pushed fanotify fd to service manager."); + + return 0; +} + +int manager_start_device_watch(Manager *manager) { + int r; + + assert(manager); + assert(manager->event); + + r = manager_init_fanotify(manager, -EBADF); + if (ERRNO_IS_NEG_NOT_SUPPORTED(r)) + return manager_start_inotify(manager); + if (r < 0) + return r; + + /* Drop unused inotify fd from fdstore. */ + manager->inotify_fd = close_and_notify_warn(manager->inotify_fd, "inotify"); + + _cleanup_(sd_event_source_unrefp) sd_event_source *s = NULL; + r = sd_event_add_io(manager->event, &s, manager->fanotify_fd, EPOLLIN, on_fanotify, manager); + if (r < 0) + return log_error_errno(r, "Failed to create event source for device watch: %m"); + + r = sd_event_source_set_priority(s, EVENT_PRIORITY_DEVICE_WATCH); + if (r < 0) + return log_error_errno(r, "Failed to set priority to event source for device watch: %m"); + + (void) sd_event_source_set_description(s, "manager-device-watch"); + + manager->device_watch_event = TAKE_PTR(s); + return 0; +} diff --git a/src/udev/udev-watch.c b/src/udev/udev-watch-inotify.c similarity index 96% rename from src/udev/udev-watch.c rename to src/udev/udev-watch-inotify.c index 73ac8a5ff7b..d34c5bc7710 100644 --- a/src/udev/udev-watch.c +++ b/src/udev/udev-watch-inotify.c @@ -52,9 +52,15 @@ static int device_new_from_watch_handle_at(sd_device **ret, int dirfd, int wd) { return sd_device_new_from_device_id(ret, id); } -void udev_watch_dump(void) { +void udev_watch_dump(Manager *manager) { int r; + assert(manager); + + /* This is inotify specific. Do nothing if inotify is disabled. */ + if (manager->inotify_fd < 0) + return; + _cleanup_closedir_ DIR *dir = opendir("/run/udev/watch/"); if (!dir) return (void) log_full_errno(errno == ENOENT ? LOG_DEBUG : LOG_WARNING, errno, @@ -276,7 +282,7 @@ int manager_start_inotify(Manager *manager) { if (r < 0) return r; - udev_watch_dump(); + udev_watch_dump(manager); _cleanup_(sd_event_source_unrefp) sd_event_source *s = NULL; r = sd_event_add_io(manager->event, &s, manager->inotify_fd, EPOLLIN, on_inotify, manager); @@ -533,20 +539,12 @@ static int notify_and_wait_signal(UdevWorker *worker, sd_device *dev, const char return sd_event_loop(e); } -void udev_watch_begin(UdevWorker *worker, sd_device *dev) { +void inotify_watch_begin(UdevWorker *worker, sd_device *dev) { int r; assert(worker); assert(dev); - /* Ignore the request of watching the device node on remove event, as the device node specified by - * DEVNAME= has already been removed, and may already be assigned to another device. Consider the - * case e.g. a USB stick memory was unplugged and then another one is plugged. */ - if (device_for_action(dev, SD_DEVICE_REMOVE)) { - log_device_debug(dev, "Not adding device watch on remove uevent."); - return; - } - r = notify_and_wait_signal(worker, dev, "INOTIFY_WATCH_ADD=1"); if (r < 0) log_device_warning_errno(dev, r, "Failed to add device watch, ignoring: %m"); @@ -554,7 +552,7 @@ void udev_watch_begin(UdevWorker *worker, sd_device *dev) { log_device_debug(dev, "Added device watch."); } -void udev_watch_end(UdevWorker *worker, sd_device *dev) { +void inotify_watch_end(UdevWorker *worker, sd_device *dev) { int r; assert(worker); diff --git a/src/udev/udev-watch.h b/src/udev/udev-watch.h index 227a1935af6..2e0ce913cbd 100644 --- a/src/udev/udev-watch.h +++ b/src/udev/udev-watch.h @@ -3,13 +3,18 @@ #include "udev-forward.h" -void udev_watch_dump(void); - +/* inotify backend */ +// FIXME: drop support for inotify once fanotify is in our baseline, i.e. 5.13 +void udev_watch_dump(Manager *manager); int manager_init_inotify(Manager *manager, int fd); int manager_start_inotify(Manager *manager); - int manager_add_watch(Manager *manager, sd_device *dev); int manager_remove_watch(Manager *manager, sd_device *dev); +void inotify_watch_begin(UdevWorker *worker, sd_device *dev); +void inotify_watch_end(UdevWorker *worker, sd_device *dev); +/* fanotify backend */ void udev_watch_begin(UdevWorker *worker, sd_device *dev); void udev_watch_end(UdevWorker *worker, sd_device *dev); +int manager_init_fanotify(Manager *manager, int fd); +int manager_start_device_watch(Manager *manager); diff --git a/src/udev/udev-worker.h b/src/udev/udev-worker.h index fa67d60b5fc..35f5a3bbfd6 100644 --- a/src/udev/udev-worker.h +++ b/src/udev/udev-worker.h @@ -17,7 +17,8 @@ typedef struct UdevWorker { UdevConfig config; - pid_t manager_pid; + pid_t manager_pid; /* used by inotify backed device watch */ + int fanotify_fd; } UdevWorker; void udev_worker_done(UdevWorker *worker); diff --git a/test/units/TEST-17-UDEV.watch-fanotify.sh b/test/units/TEST-17-UDEV.watch-fanotify.sh new file mode 100755 index 00000000000..9b47d9693d2 --- /dev/null +++ b/test/units/TEST-17-UDEV.watch-fanotify.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: LGPL-2.1-or-later +set -ex +set -o pipefail + +# tests for udev watch + +# shellcheck source=test/units/util.sh +. "$(dirname "$0")"/util.sh + +udevd_fanotify_fdinfo() { + local pid fd + + # Print the path of systemd-udevd's fanotify group fdinfo. + + pid="$(systemctl show --property MainPID --value systemd-udevd.service)" + [[ "${pid:-0}" -gt 0 ]] || return 1 + + for fd in /proc/"$pid"/fd/*; do + if [[ "$(readlink "$fd" 2>/dev/null)" == "anon_inode:[fanotify]" ]]; then + echo "/proc/$pid/fdinfo/${fd##*/}" + return 0 + fi + done + + return 1 +} + +device_is_watched() { + local devnode="${1:?}" + local ino fdinfo + + fdinfo="$(udevd_fanotify_fdinfo)" || exit 1 + ino="$(printf '%x' "$(stat -c '%i' "$devnode")")" + grep -qE "^fanotify ino:${ino} " "$fdinfo" +} + +ROOTDEV="$(bootctl -RR)" +ROOTDEV_NAME="$(udevadm info --query=name "$ROOTDEV")" + +mkdir -p /run/udev/rules.d/ + +cat >/run/udev/rules.d/99-testsuite.rules </run/udev/rules.d/99-testsuite.rules </run/systemd/system/systemd-udevd.service.d/10-debug.conf <