journald: expose message counters as metrics

journald now implements io.systemd.Metrics on
/run/systemd/report/io.systemd.JournalDaemon, via the new
systemd-journald-metrics.socket, and reports the message counters as
io.systemd.JournalDaemon.MessagesByPriority (field priority=) and
io.systemd.JournalDaemon.MessagesByTransport (field transport=).

Metrics requests are processed at a lower event loop priority than
incoming log messages. For now only the main journald instance provides
metrics, not the namespace instances.

Like the audit socket, the metrics socket is not listed in Sockets= of
the service, so that it can be disabled.

man/rules/meson.build

@@ -1120,6 +1120,7 @@ manpages = [
   ['systemd-journald',
    'systemd-journald-audit.socket',
    'systemd-journald-dev-log.socket',
+   'systemd-journald-metrics.socket',
    'systemd-journald-varlink@.socket',
    'systemd-journald.socket',
    'systemd-journald@.service',

man/systemd-journald.service.xml

@@ -21,6 +21,7 @@
     <refname>systemd-journald.socket</refname>
     <refname>systemd-journald-dev-log.socket</refname>
     <refname>systemd-journald-audit.socket</refname>
+    <refname>systemd-journald-metrics.socket</refname>
     <refname>systemd-journald@.service</refname>
     <refname>systemd-journald@.socket</refname>
     <refname>systemd-journald-varlink@.socket</refname>
@@ -34,6 +35,7 @@
       <member><filename>systemd-journald.socket</filename></member>
       <member><filename>systemd-journald-dev-log.socket</filename></member>
       <member><filename>systemd-journald-audit.socket</filename></member>
+      <member><filename>systemd-journald-metrics.socket</filename></member>
       <member><filename>systemd-journald@.service</filename></member>
       <member><filename>systemd-journald@.socket</filename></member>
       <member><filename>systemd-journald-varlink@.socket</filename></member>
@@ -352,6 +354,47 @@ systemd-tmpfiles --create --prefix /var/log/journal</programlisting>
     <filename>/var/log/journal/</filename> directory.</para>
   </refsect1>
 
+  <refsect1>
+    <title>Metrics</title>
+
+    <para><command>systemd-journald</command> is a metrics source for
+    <citerefentry><refentrytitle>systemd-report</refentrytitle><manvolnum>1</manvolnum></citerefentry>. If
+    <filename>systemd-journald-metrics.socket</filename> is enabled, it implements the
+    <constant>io.systemd.Metrics</constant> Varlink interface on
+    <filename>/run/systemd/report/io.systemd.JournalDaemon</filename>, and reports the following counter
+    metrics:</para>
+
+    <variablelist>
+      <varlistentry>
+        <term><literal>io.systemd.JournalDaemon.MessagesByPriority</literal></term>
+        <listitem><para>The number of log messages received since boot, by priority. The metric field
+        <varname>priority</varname> is set to one of <literal>emerg</literal>, <literal>alert</literal>,
+        <literal>crit</literal>, <literal>err</literal>, <literal>warning</literal>,
+        <literal>notice</literal>, <literal>info</literal>, <literal>debug</literal>.</para>
+
+        <xi:include href="version-info.xml" xpointer="v263"/></listitem>
+      </varlistentry>
+
+      <varlistentry>
+        <term><literal>io.systemd.JournalDaemon.MessagesByTransport</literal></term>
+        <listitem><para>The number of log messages received since boot, by transport. The metric field
+        <varname>transport</varname> is set to one of <literal>syslog</literal>,
+        <literal>journal</literal>, <literal>stdout</literal>, <literal>audit</literal>,
+        <literal>kernel</literal>, matching the <varname>_TRANSPORT=</varname> journal field, see
+        <citerefentry><refentrytitle>systemd.journal-fields</refentrytitle><manvolnum>7</manvolnum></citerefentry>.</para>
+
+        <xi:include href="version-info.xml" xpointer="v263"/></listitem>
+      </varlistentry>
+    </variablelist>
+
+    <para>Messages are counted when they are received, before any filtering (such as
+    <varname>LogLevelMax=</varname>, <varname>LogFilterPatterns=</varname>, <varname>MaxLevelStore=</varname>,
+    <varname>Storage=none</varname>) or rate limiting is applied. The counters hence include messages that
+    are not stored in the journal. Messages generated by <command>systemd-journald</command> itself are
+    not counted. The counters are kept across restarts of the service, but reset on reboot. Only the
+    default journal namespace provides these metrics.</para>
+  </refsect1>
+
   <refsect1>
     <title>Files</title>
 
@@ -422,6 +465,15 @@ systemd-tmpfiles --create --prefix /var/log/journal</programlisting>
 
         <xi:include href="version-info.xml" xpointer="v228"/></listitem>
       </varlistentry>
+
+      <varlistentry>
+        <term><filename>/run/systemd/journal/counters</filename></term>
+
+        <listitem><para>The message counters reported via the metrics described above. This file is
+        internal to <command>systemd-journald</command> and not a stable interface.</para>
+
+        <xi:include href="version-info.xml" xpointer="v263"/></listitem>
+      </varlistentry>
     </variablelist>
 
     <para>If journal namespacing is used these paths are slightly altered to include a namespace identifier, see above.</para>

presets/90-systemd.preset

@@ -27,6 +27,7 @@ enable systemd-homed.service
 enable systemd-homed-activate.service
 enable systemd-journalctl-metrics.socket
 enable systemd-journald-audit.socket
+enable systemd-journald-metrics.socket
 enable systemd-mountfsd.socket
 enable systemd-network-generator.service
 enable systemd-networkd.service

src/journal/journald-manager.c

@@ -40,6 +40,7 @@
 #include "journald-counters.h"
 #include "journald-kmsg.h"
 #include "journald-manager.h"
+#include "journald-metrics.h"
 #include "journald-native.h"
 #include "journald-rate-limit.h"
 #include "journald-socket.h"
@@ -2333,7 +2334,7 @@ int manager_new(Manager **ret) {
 int manager_init(Manager *m) {
         const char *native_socket, *syslog_socket, *stdout_socket, *varlink_socket, *e;
         _cleanup_fdset_free_ FDSet *fds = NULL;
-        int n, r, varlink_fd = -EBADF;
+        int n, r, varlink_fd = -EBADF, metrics_fd = -EBADF;
         bool no_sockets;
 
         assert(m);
@@ -2410,6 +2411,13 @@ int manager_init(Manager *m) {
                                                        "Too many varlink sockets passed.");
 
                         varlink_fd = fd;
+                } else if (sd_is_socket_unix(fd, SOCK_STREAM, 1, JOURNALD_METRICS_SOCKET, 0) > 0) {
+
+                        if (metrics_fd >= 0)
+                                return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                                                       "Too many metrics varlink sockets passed.");
+
+                        metrics_fd = fd;
                 } else if (sd_is_socket(fd, AF_NETLINK, SOCK_RAW, -1) > 0) {
 
                         if (m->audit_fd >= 0)
@@ -2477,6 +2485,10 @@ int manager_init(Manager *m) {
         if (r < 0)
                 return r;
 
+        r = manager_open_metrics(m, metrics_fd);
+        if (r < 0)
+                return r;
+
         r = manager_map_seqnum_file(m, "seqnum", sizeof(SeqnumData), (void**) &m->seqnum);
         if (r < 0)
                 return log_error_errno(r, "Failed to map main seqnum file: %m");
@@ -2568,6 +2580,7 @@ Manager* manager_free(Manager *m) {
         ordered_hashmap_free(m->user_journals);
 
         sd_varlink_server_unref(m->varlink_server);
+        sd_varlink_server_unref(m->metrics_varlink_server);
 
         sd_event_source_unref(m->syslog_event_source);
         sd_event_source_unref(m->native_event_source);

src/journal/journald-manager.h

@@ -121,6 +121,7 @@ typedef struct Manager {
         ClientContext *pid1_context; /* the context of PID 1 */
 
         sd_varlink_server *varlink_server;
+        sd_varlink_server *metrics_varlink_server;
 
         /* timestamp of most recently processed log messages from each source (CLOCK_REALTIME for the first
          * two, CLOCK_BOOTTIME for the other) */

src/journal/journald-metrics.c

@@ -0,0 +1,146 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include "sd-event.h"
+#include "sd-json.h"
+#include "sd-varlink.h"
+
+#include "alloc-util.h"
+#include "argv-util.h"
+#include "errno-util.h"
+#include "fd-util.h"
+#include "journald-counters.h"
+#include "journald-manager.h"
+#include "journald-metrics.h"
+#include "log.h"
+#include "metrics.h"
+#include "syslog-util.h"
+
+#define METRIC_IO_SYSTEMD_JOURNAL_DAEMON_PREFIX "io.systemd.JournalDaemon."
+
+static int messages_by_priority_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) {
+        Manager *m = ASSERT_PTR(userdata);
+        int r;
+
+        assert(mf);
+        assert(link);
+
+        if (!m->counters)
+                return 0;
+
+        for (int i = 0; i < (int) ELEMENTSOF(m->counters->n_priority); i++) {
+                _cleanup_free_ char *s = NULL;
+                r = log_level_to_string_alloc(i, &s);
+                if (r < 0)
+                        return r;
+
+                _cleanup_(sd_json_variant_unrefp) sd_json_variant *fields = NULL;
+                r = sd_json_buildo(&fields, SD_JSON_BUILD_PAIR_STRING("priority", s));
+                if (r < 0)
+                        return r;
+
+                r = metric_build_send_unsigned(mf, link, /* object= */ NULL, m->counters->n_priority[i], fields);
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}
+
+static int messages_by_transport_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) {
+        Manager *m = ASSERT_PTR(userdata);
+        int r;
+
+        assert(mf);
+        assert(link);
+
+        if (!m->counters)
+                return 0;
+
+        for (JournalTransport t = 0; t < _JOURNAL_TRANSPORT_MAX; t++) {
+                _cleanup_(sd_json_variant_unrefp) sd_json_variant *fields = NULL;
+                r = sd_json_buildo(&fields, SD_JSON_BUILD_PAIR_STRING("transport", journal_transport_to_string(t)));
+                if (r < 0)
+                        return r;
+
+                r = metric_build_send_unsigned(mf, link, /* object= */ NULL, m->counters->n_transport[t], fields);
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}
+
+static const MetricFamily journald_metric_family_table[] = {
+        /* Keep metrics ordered alphabetically */
+        {
+                .name = METRIC_IO_SYSTEMD_JOURNAL_DAEMON_PREFIX "MessagesByPriority",
+                .description = "Number of log messages received since boot, by priority (priority=emerg|alert|crit|err|warning|notice|info|debug)",
+                .type = METRIC_FAMILY_TYPE_COUNTER,
+                .generate = messages_by_priority_generate,
+        },
+        {
+                .name = METRIC_IO_SYSTEMD_JOURNAL_DAEMON_PREFIX "MessagesByTransport",
+                .description = "Number of log messages received since boot, by transport (transport=syslog|journal|stdout|audit|kernel)",
+                .type = METRIC_FAMILY_TYPE_COUNTER,
+                .generate = messages_by_transport_generate,
+        },
+        {}
+};
+
+static int vl_method_metrics_describe(sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata) {
+        return metrics_method_describe(journald_metric_family_table, link, parameters, flags, userdata);
+}
+
+static int vl_method_metrics_list(sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata) {
+        return metrics_method_list(journald_metric_family_table, link, parameters, flags, userdata);
+}
+
+int manager_open_metrics(Manager *m, int fd) {
+        _unused_ _cleanup_close_ int fd_close = fd; /* take possession */
+        int r;
+
+        assert(m);
+
+        if (m->metrics_varlink_server)
+                return 0;
+
+        /* Only the main instance serves metrics for now */
+        if (m->namespace)
+                return 0;
+
+        if (fd < 0 && invoked_by_systemd()) {
+                log_debug("systemd-journald-metrics.socket seems to be disabled, not installing metrics varlink server.");
+                return 0;
+        }
+
+        /* Process metrics requests at a lower priority than incoming log messages */
+        _cleanup_(sd_varlink_server_unrefp) sd_varlink_server *s = NULL;
+        r = metrics_setup_varlink_server(
+                        &s,
+                        SD_VARLINK_SERVER_ACCOUNT_UID|SD_VARLINK_SERVER_INHERIT_USERDATA,
+                        m->event,
+                        SD_EVENT_PRIORITY_NORMAL+10,
+                        vl_method_metrics_list,
+                        vl_method_metrics_describe,
+                        m);
+        if (r < 0)
+                return log_error_errno(r, "Failed to set up metrics varlink server: %m");
+
+        if (fd < 0) {
+                r = sd_varlink_server_listen_address(
+                                s,
+                                JOURNALD_METRICS_SOCKET,
+                                0666 | SD_VARLINK_SERVER_MODE_MKDIR_0755);
+                if (ERRNO_IS_NEG_PRIVILEGE(r)) {
+                        log_warning_errno(r, "Failed to bind to metrics varlink socket, ignoring: %m");
+                        return 0;
+                }
+        } else
+                r = sd_varlink_server_listen_fd(s, fd);
+        if (r < 0)
+                return log_error_errno(r, "Failed to bind to metrics varlink socket: %m");
+
+        TAKE_FD(fd_close);
+        m->metrics_varlink_server = TAKE_PTR(s);
+        return 0;
+}

src/journal/journald-metrics.h

@@ -0,0 +1,8 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include "journald-forward.h"
+
+#define JOURNALD_METRICS_SOCKET "/run/systemd/report/io.systemd.JournalDaemon"
+
+int manager_open_metrics(Manager *m, int fd);

src/journal/meson.build

@@ -13,6 +13,7 @@ systemd_journald_export_sources = files(
         'journald-counters.c',
         'journald-kmsg.c',
         'journald-manager.c',
+        'journald-metrics.c',
         'journald-native.c',
         'journald-rate-limit.c',
         'journald-socket.c',

units/meson.build

@@ -519,6 +519,7 @@ units = [
           'file' : 'systemd-journald-dev-log.socket',
           'symlinks' : ['sockets.target.wants/'],
         },
+        { 'file' : 'systemd-journald-metrics.socket' },
         { 'file' : 'systemd-journald-sync@.service' },
         { 'file' : 'systemd-journald-varlink@.socket' },
         {

units/systemd-journald-metrics.socket

@@ -0,0 +1,26 @@
+#  SPDX-License-Identifier: LGPL-2.1-or-later
+#
+#  This file is part of systemd.
+#
+#  systemd is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU Lesser General Public License as published by
+#  the Free Software Foundation; either version 2.1 of the License, or
+#  (at your option) any later version.
+
+[Unit]
+Description=Journal Metrics Varlink Socket
+Documentation=man:systemd-journald.service(8)
+DefaultDependencies=no
+Before=sockets.target
+IgnoreOnIsolate=yes
+
+[Socket]
+Service=systemd-journald.service
+ListenStream=/run/systemd/report/io.systemd.JournalDaemon
+SocketMode=0666
+DeferTrigger=yes
+XAttrEntryPoint=user.varlink=entrypoint
+XAttrListen=user.varlink=listen
+
+[Install]
+WantedBy=sockets.target

units/systemd-journald.service.in

@@ -49,9 +49,9 @@ RestrictRealtime=yes
 RestrictSUIDSGID=yes
 RuntimeDirectory=systemd/journal
 RuntimeDirectoryPreserve=yes
-# Audit socket is not listed here because this unit can be turned off. However
-# the link between the socket and the service units is still created thanks to
-# the 'Service=' setting specified in the socket unit.
+# Audit and metrics sockets are not listed here because these units can be turned
+# off. However the link between the socket and the service units is still created
+# thanks to the 'Service=' setting specified in the socket units.
 Sockets=systemd-journald.socket systemd-journald-dev-log.socket
 StandardOutput=null
 SystemCallArchitectures=native