diff --git a/man/rules/meson.build b/man/rules/meson.build index cd95cbf1dcb..8c5a6abbfa7 100644 --- a/man/rules/meson.build +++ b/man/rules/meson.build @@ -1120,6 +1120,7 @@ manpages = [ ['systemd-journald', 'systemd-journald-audit.socket', 'systemd-journald-dev-log.socket', + 'systemd-journald-metrics.socket', 'systemd-journald-varlink@.socket', 'systemd-journald.socket', 'systemd-journald@.service', diff --git a/man/systemd-journald.service.xml b/man/systemd-journald.service.xml index eb84a311338..f3e4a524883 100644 --- a/man/systemd-journald.service.xml +++ b/man/systemd-journald.service.xml @@ -21,6 +21,7 @@ systemd-journald.socket systemd-journald-dev-log.socket systemd-journald-audit.socket + systemd-journald-metrics.socket systemd-journald@.service systemd-journald@.socket systemd-journald-varlink@.socket @@ -34,6 +35,7 @@ systemd-journald.socket systemd-journald-dev-log.socket systemd-journald-audit.socket + systemd-journald-metrics.socket systemd-journald@.service systemd-journald@.socket systemd-journald-varlink@.socket @@ -352,6 +354,47 @@ systemd-tmpfiles --create --prefix /var/log/journal /var/log/journal/ directory. + + Metrics + + systemd-journald is a metrics source for + systemd-report1. If + systemd-journald-metrics.socket is enabled, it implements the + io.systemd.Metrics Varlink interface on + /run/systemd/report/io.systemd.JournalDaemon, and reports the following counter + metrics: + + + + io.systemd.JournalDaemon.MessagesByPriority + The number of log messages received since boot, by priority. The metric field + priority is set to one of emerg, alert, + crit, err, warning, + notice, info, debug. + + + + + + io.systemd.JournalDaemon.MessagesByTransport + The number of log messages received since boot, by transport. The metric field + transport is set to one of syslog, + journal, stdout, audit, + kernel, matching the _TRANSPORT= journal field, see + systemd.journal-fields7. + + + + + + Messages are counted when they are received, before any filtering (such as + LogLevelMax=, LogFilterPatterns=, MaxLevelStore=, + Storage=none) or rate limiting is applied. The counters hence include messages that + are not stored in the journal. Messages generated by systemd-journald itself are + not counted. The counters are kept across restarts of the service, but reset on reboot. Only the + default journal namespace provides these metrics. + + Files @@ -422,6 +465,15 @@ systemd-tmpfiles --create --prefix /var/log/journal + + + /run/systemd/journal/counters + + The message counters reported via the metrics described above. This file is + internal to systemd-journald and not a stable interface. + + + If journal namespacing is used these paths are slightly altered to include a namespace identifier, see above. diff --git a/presets/90-systemd.preset b/presets/90-systemd.preset index 71709be7680..9a67c65c72a 100644 --- a/presets/90-systemd.preset +++ b/presets/90-systemd.preset @@ -27,6 +27,7 @@ enable systemd-homed.service enable systemd-homed-activate.service enable systemd-journalctl-metrics.socket enable systemd-journald-audit.socket +enable systemd-journald-metrics.socket enable systemd-mountfsd.socket enable systemd-network-generator.service enable systemd-networkd.service diff --git a/src/journal/journald-manager.c b/src/journal/journald-manager.c index 9b6da0477d7..7c43a93ed8a 100644 --- a/src/journal/journald-manager.c +++ b/src/journal/journald-manager.c @@ -40,6 +40,7 @@ #include "journald-counters.h" #include "journald-kmsg.h" #include "journald-manager.h" +#include "journald-metrics.h" #include "journald-native.h" #include "journald-rate-limit.h" #include "journald-socket.h" @@ -2333,7 +2334,7 @@ int manager_new(Manager **ret) { int manager_init(Manager *m) { const char *native_socket, *syslog_socket, *stdout_socket, *varlink_socket, *e; _cleanup_fdset_free_ FDSet *fds = NULL; - int n, r, varlink_fd = -EBADF; + int n, r, varlink_fd = -EBADF, metrics_fd = -EBADF; bool no_sockets; assert(m); @@ -2410,6 +2411,13 @@ int manager_init(Manager *m) { "Too many varlink sockets passed."); varlink_fd = fd; + } else if (sd_is_socket_unix(fd, SOCK_STREAM, 1, JOURNALD_METRICS_SOCKET, 0) > 0) { + + if (metrics_fd >= 0) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Too many metrics varlink sockets passed."); + + metrics_fd = fd; } else if (sd_is_socket(fd, AF_NETLINK, SOCK_RAW, -1) > 0) { if (m->audit_fd >= 0) @@ -2477,6 +2485,10 @@ int manager_init(Manager *m) { if (r < 0) return r; + r = manager_open_metrics(m, metrics_fd); + if (r < 0) + return r; + r = manager_map_seqnum_file(m, "seqnum", sizeof(SeqnumData), (void**) &m->seqnum); if (r < 0) return log_error_errno(r, "Failed to map main seqnum file: %m"); @@ -2568,6 +2580,7 @@ Manager* manager_free(Manager *m) { ordered_hashmap_free(m->user_journals); sd_varlink_server_unref(m->varlink_server); + sd_varlink_server_unref(m->metrics_varlink_server); sd_event_source_unref(m->syslog_event_source); sd_event_source_unref(m->native_event_source); diff --git a/src/journal/journald-manager.h b/src/journal/journald-manager.h index 942d42452f6..00b23652dcb 100644 --- a/src/journal/journald-manager.h +++ b/src/journal/journald-manager.h @@ -121,6 +121,7 @@ typedef struct Manager { ClientContext *pid1_context; /* the context of PID 1 */ sd_varlink_server *varlink_server; + sd_varlink_server *metrics_varlink_server; /* timestamp of most recently processed log messages from each source (CLOCK_REALTIME for the first * two, CLOCK_BOOTTIME for the other) */ diff --git a/src/journal/journald-metrics.c b/src/journal/journald-metrics.c new file mode 100644 index 00000000000..f144a6783d3 --- /dev/null +++ b/src/journal/journald-metrics.c @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include "sd-event.h" +#include "sd-json.h" +#include "sd-varlink.h" + +#include "alloc-util.h" +#include "argv-util.h" +#include "errno-util.h" +#include "fd-util.h" +#include "journald-counters.h" +#include "journald-manager.h" +#include "journald-metrics.h" +#include "log.h" +#include "metrics.h" +#include "syslog-util.h" + +#define METRIC_IO_SYSTEMD_JOURNAL_DAEMON_PREFIX "io.systemd.JournalDaemon." + +static int messages_by_priority_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) { + Manager *m = ASSERT_PTR(userdata); + int r; + + assert(mf); + assert(link); + + if (!m->counters) + return 0; + + for (int i = 0; i < (int) ELEMENTSOF(m->counters->n_priority); i++) { + _cleanup_free_ char *s = NULL; + r = log_level_to_string_alloc(i, &s); + if (r < 0) + return r; + + _cleanup_(sd_json_variant_unrefp) sd_json_variant *fields = NULL; + r = sd_json_buildo(&fields, SD_JSON_BUILD_PAIR_STRING("priority", s)); + if (r < 0) + return r; + + r = metric_build_send_unsigned(mf, link, /* object= */ NULL, m->counters->n_priority[i], fields); + if (r < 0) + return r; + } + + return 0; +} + +static int messages_by_transport_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) { + Manager *m = ASSERT_PTR(userdata); + int r; + + assert(mf); + assert(link); + + if (!m->counters) + return 0; + + for (JournalTransport t = 0; t < _JOURNAL_TRANSPORT_MAX; t++) { + _cleanup_(sd_json_variant_unrefp) sd_json_variant *fields = NULL; + r = sd_json_buildo(&fields, SD_JSON_BUILD_PAIR_STRING("transport", journal_transport_to_string(t))); + if (r < 0) + return r; + + r = metric_build_send_unsigned(mf, link, /* object= */ NULL, m->counters->n_transport[t], fields); + if (r < 0) + return r; + } + + return 0; +} + +static const MetricFamily journald_metric_family_table[] = { + /* Keep metrics ordered alphabetically */ + { + .name = METRIC_IO_SYSTEMD_JOURNAL_DAEMON_PREFIX "MessagesByPriority", + .description = "Number of log messages received since boot, by priority (priority=emerg|alert|crit|err|warning|notice|info|debug)", + .type = METRIC_FAMILY_TYPE_COUNTER, + .generate = messages_by_priority_generate, + }, + { + .name = METRIC_IO_SYSTEMD_JOURNAL_DAEMON_PREFIX "MessagesByTransport", + .description = "Number of log messages received since boot, by transport (transport=syslog|journal|stdout|audit|kernel)", + .type = METRIC_FAMILY_TYPE_COUNTER, + .generate = messages_by_transport_generate, + }, + {} +}; + +static int vl_method_metrics_describe(sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata) { + return metrics_method_describe(journald_metric_family_table, link, parameters, flags, userdata); +} + +static int vl_method_metrics_list(sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata) { + return metrics_method_list(journald_metric_family_table, link, parameters, flags, userdata); +} + +int manager_open_metrics(Manager *m, int fd) { + _unused_ _cleanup_close_ int fd_close = fd; /* take possession */ + int r; + + assert(m); + + if (m->metrics_varlink_server) + return 0; + + /* Only the main instance serves metrics for now */ + if (m->namespace) + return 0; + + if (fd < 0 && invoked_by_systemd()) { + log_debug("systemd-journald-metrics.socket seems to be disabled, not installing metrics varlink server."); + return 0; + } + + /* Process metrics requests at a lower priority than incoming log messages */ + _cleanup_(sd_varlink_server_unrefp) sd_varlink_server *s = NULL; + r = metrics_setup_varlink_server( + &s, + SD_VARLINK_SERVER_ACCOUNT_UID|SD_VARLINK_SERVER_INHERIT_USERDATA, + m->event, + SD_EVENT_PRIORITY_NORMAL+10, + vl_method_metrics_list, + vl_method_metrics_describe, + m); + if (r < 0) + return log_error_errno(r, "Failed to set up metrics varlink server: %m"); + + if (fd < 0) { + r = sd_varlink_server_listen_address( + s, + JOURNALD_METRICS_SOCKET, + 0666 | SD_VARLINK_SERVER_MODE_MKDIR_0755); + if (ERRNO_IS_NEG_PRIVILEGE(r)) { + log_warning_errno(r, "Failed to bind to metrics varlink socket, ignoring: %m"); + return 0; + } + } else + r = sd_varlink_server_listen_fd(s, fd); + if (r < 0) + return log_error_errno(r, "Failed to bind to metrics varlink socket: %m"); + + TAKE_FD(fd_close); + m->metrics_varlink_server = TAKE_PTR(s); + return 0; +} diff --git a/src/journal/journald-metrics.h b/src/journal/journald-metrics.h new file mode 100644 index 00000000000..b46547b8cfb --- /dev/null +++ b/src/journal/journald-metrics.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include "journald-forward.h" + +#define JOURNALD_METRICS_SOCKET "/run/systemd/report/io.systemd.JournalDaemon" + +int manager_open_metrics(Manager *m, int fd); diff --git a/src/journal/meson.build b/src/journal/meson.build index 92898cf519e..94b36b85ea3 100644 --- a/src/journal/meson.build +++ b/src/journal/meson.build @@ -13,6 +13,7 @@ systemd_journald_export_sources = files( 'journald-counters.c', 'journald-kmsg.c', 'journald-manager.c', + 'journald-metrics.c', 'journald-native.c', 'journald-rate-limit.c', 'journald-socket.c', diff --git a/units/meson.build b/units/meson.build index e01644e05a3..2f2cfb632c3 100644 --- a/units/meson.build +++ b/units/meson.build @@ -519,6 +519,7 @@ units = [ 'file' : 'systemd-journald-dev-log.socket', 'symlinks' : ['sockets.target.wants/'], }, + { 'file' : 'systemd-journald-metrics.socket' }, { 'file' : 'systemd-journald-sync@.service' }, { 'file' : 'systemd-journald-varlink@.socket' }, { diff --git a/units/systemd-journald-metrics.socket b/units/systemd-journald-metrics.socket new file mode 100644 index 00000000000..60e7e4a4cb3 --- /dev/null +++ b/units/systemd-journald-metrics.socket @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +# +# This file is part of systemd. +# +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=Journal Metrics Varlink Socket +Documentation=man:systemd-journald.service(8) +DefaultDependencies=no +Before=sockets.target +IgnoreOnIsolate=yes + +[Socket] +Service=systemd-journald.service +ListenStream=/run/systemd/report/io.systemd.JournalDaemon +SocketMode=0666 +DeferTrigger=yes +XAttrEntryPoint=user.varlink=entrypoint +XAttrListen=user.varlink=listen + +[Install] +WantedBy=sockets.target diff --git a/units/systemd-journald.service.in b/units/systemd-journald.service.in index 1fb080d2685..a5a04d79a9b 100644 --- a/units/systemd-journald.service.in +++ b/units/systemd-journald.service.in @@ -49,9 +49,9 @@ RestrictRealtime=yes RestrictSUIDSGID=yes RuntimeDirectory=systemd/journal RuntimeDirectoryPreserve=yes -# Audit socket is not listed here because this unit can be turned off. However -# the link between the socket and the service units is still created thanks to -# the 'Service=' setting specified in the socket unit. +# Audit and metrics sockets are not listed here because these units can be turned +# off. However the link between the socket and the service units is still created +# thanks to the 'Service=' setting specified in the socket units. Sockets=systemd-journald.socket systemd-journald-dev-log.socket StandardOutput=null SystemCallArchitectures=native