From c659d0f55ee5854e40dbaccc20ad0e8da176b271 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 22 Jun 2023 18:48:41 +0200 Subject: [PATCH 1/3] userdbd: drastically raise ratelimit we apply on requests for more worker processes These requests might come in during lookup floods very quickly, since multiple worker processes might detect that things should be scaled up at the same time. Hence, let's substantially raise the limit so that it doesn't get hit in real-life scenarios and acts more like a safety net. --- src/userdb/userdbd-manager.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/userdb/userdbd-manager.c b/src/userdb/userdbd-manager.c index 8101ac52db2..1895e151f4e 100644 --- a/src/userdb/userdbd-manager.c +++ b/src/userdb/userdbd-manager.c @@ -71,8 +71,8 @@ int manager_new(Manager **ret) { *m = (Manager) { .listen_fd = -EBADF, .worker_ratelimit = { - .interval = 5 * USEC_PER_SEC, - .burst = 50, + .interval = 2 * USEC_PER_SEC, + .burst = 2500, }, }; From 3ef0103f2265a68e32847deaf84588c2fa711f3a Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 22 Jun 2023 18:50:43 +0200 Subject: [PATCH 2/3] userdbd: prefix parameters with their names more --- src/userdb/userdbd-manager.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/userdb/userdbd-manager.c b/src/userdb/userdbd-manager.c index 1895e151f4e..372de720f78 100644 --- a/src/userdb/userdbd-manager.c +++ b/src/userdb/userdbd-manager.c @@ -49,7 +49,7 @@ static int on_sigusr2(sd_event_source *s, const struct signalfd_siginfo *si, voi assert(s); - (void) start_workers(m, true); /* Workers told us there's more work, let's add one more worker as long as we are below the high watermark */ + (void) start_workers(m, /* explicit_request=*/ true); /* Workers told us there's more work, let's add one more worker as long as we are below the high watermark */ return 0; } @@ -281,5 +281,5 @@ int manager_startup(Manager *m) { if (setsockopt(m->listen_fd, SOL_SOCKET, SO_RCVTIMEO, TIMEVAL_STORE(LISTEN_TIMEOUT_USEC), sizeof(struct timeval)) < 0) return log_error_errno(errno, "Failed to se SO_RCVTIMEO: %m"); - return start_workers(m, false); + return start_workers(m, /* explicit_request= */ false); } From 2543deada391095887dd5b4fdefe3e66f76e32a9 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 22 Jun 2023 18:51:34 +0200 Subject: [PATCH 3/3] userdbd: when we hit a flood of requests to start more workers, don't exit Let's tweak what we do if we detect a flood of requests to start more workers: if none of the workers ever sticks (i.e. the worker count is zero) then let's just give up, as before. Otherwise, let's just not start more workers for a while, and do so again after a while. Thus spawning ofr workers will "cool off" for a while. Fixes: #27028 --- src/userdb/userdbd-manager.c | 40 +++++++++++++++++++++++++++++++++--- src/userdb/userdbd-manager.h | 2 ++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/userdb/userdbd-manager.c b/src/userdb/userdbd-manager.c index 372de720f78..73da3fb1e8c 100644 --- a/src/userdb/userdbd-manager.c +++ b/src/userdb/userdbd-manager.c @@ -53,6 +53,17 @@ static int on_sigusr2(sd_event_source *s, const struct signalfd_siginfo *si, voi return 0; } +static int on_deferred_start_worker(sd_event_source *s, uint64_t usec, void *userdata) { + Manager *m = ASSERT_PTR(userdata); + + assert(s); + + m->deferred_start_worker_event_source = sd_event_source_unref(m->deferred_start_worker_event_source); + + (void) start_workers(m, /* explicit_request=*/ false); + return 0; +} + DEFINE_HASH_OPS_WITH_KEY_DESTRUCTOR( event_source_hash_ops, sd_event_source, @@ -111,6 +122,8 @@ Manager* manager_free(Manager *m) { set_free(m->workers_fixed); set_free(m->workers_dynamic); + m->deferred_start_worker_event_source = sd_event_source_unref(m->deferred_start_worker_event_source); + sd_event_unref(m->event); return mfree(m); @@ -213,10 +226,31 @@ static int start_workers(Manager *m, bool explicit_request) { break; if (!ratelimit_below(&m->worker_ratelimit)) { - /* If we keep starting workers too often, let's fail the whole daemon, something is wrong */ - sd_event_exit(m->event, EXIT_FAILURE); - return log_error_errno(SYNTHETIC_ERRNO(EUCLEAN), "Worker threads requested too frequently, something is wrong."); + /* If we keep starting workers too often but none sticks, let's fail the whole + * daemon, something is wrong */ + if (n == 0) { + sd_event_exit(m->event, EXIT_FAILURE); + return log_error_errno(SYNTHETIC_ERRNO(EUCLEAN), "Worker threads requested too frequently, but worker count is zero, something is wrong."); + } + + /* Otherwise, let's stop spawning more for a while. */ + log_warning("Worker threads requested too frequently, not starting new ones for a while."); + + if (!m->deferred_start_worker_event_source) { + r = sd_event_add_time( + m->event, + &m->deferred_start_worker_event_source, + CLOCK_MONOTONIC, + ratelimit_end(&m->worker_ratelimit), + /* accuracy_usec= */ 0, + on_deferred_start_worker, + m); + if (r < 0) + return log_error_errno(r, "Failed to allocate deferred start worker event source: %m"); + } + + break; } r = start_one_worker(m); diff --git a/src/userdb/userdbd-manager.h b/src/userdb/userdbd-manager.h index 4fb45e6c356..c39f79d25ca 100644 --- a/src/userdb/userdbd-manager.h +++ b/src/userdb/userdbd-manager.h @@ -21,6 +21,8 @@ struct Manager { int listen_fd; RateLimit worker_ratelimit; + + sd_event_source *deferred_start_worker_event_source; }; int manager_new(Manager **ret);