sd-journal: introduce sd_journal_get_data_to_fd()

It may be useful for getting large data, e.g. coredump, from journal.

man/rules/meson.build

@@ -772,10 +772,12 @@ manpages = [
   ''],
  ['sd_journal_get_data',
   '3',
-  ['SD_JOURNAL_FOREACH_DATA',
+  ['SD_JOURNAL_DATA_SKIP_FIELD',
+   'SD_JOURNAL_FOREACH_DATA',
    'sd_journal_enumerate_available_data',
    'sd_journal_enumerate_data',
    'sd_journal_get_data_threshold',
+   'sd_journal_get_data_to_fd',
    'sd_journal_restart_data',
    'sd_journal_set_data_threshold'],
   ''],

man/sd_journal_get_data.xml

@@ -17,12 +17,14 @@
 
   <refnamediv>
     <refname>sd_journal_get_data</refname>
+    <refname>sd_journal_get_data_to_fd</refname>
     <refname>sd_journal_enumerate_data</refname>
     <refname>sd_journal_enumerate_available_data</refname>
     <refname>sd_journal_restart_data</refname>
     <refname>SD_JOURNAL_FOREACH_DATA</refname>
     <refname>sd_journal_set_data_threshold</refname>
     <refname>sd_journal_get_data_threshold</refname>
+    <refname>SD_JOURNAL_DATA_SKIP_FIELD</refname>
     <refpurpose>Read data fields from the current journal entry</refpurpose>
   </refnamediv>
 
@@ -38,6 +40,15 @@
         <paramdef>size_t *<parameter>length</parameter></paramdef>
       </funcprototype>
 
+      <funcprototype>
+        <funcdef>int <function>sd_journal_get_data_to_fd</function></funcdef>
+        <paramdef>sd_journal *<parameter>j</parameter></paramdef>
+        <paramdef>const char *<parameter>field</parameter></paramdef>
+        <paramdef>sd_journal_data_flags_t <parameter>flags</parameter></paramdef>
+        <paramdef>int <parameter>fd</parameter></paramdef>
+        <paramdef>size_t *<parameter>ret_size</parameter></paramdef>
+      </funcprototype>
+
       <funcprototype>
         <funcdef>int <function>sd_journal_enumerate_data</function></funcdef>
         <paramdef>sd_journal *<parameter>j</parameter></paramdef>
@@ -94,6 +105,14 @@
     <function>sd_journal_enumerate_available_data()</function>, or when the read pointer is altered. Note
     that the data returned will be prefixed with the field name and <literal>=</literal>.</para>
 
+    <para><function>sd_journal_get_data_to_fd()</function> is similar to
+    <function>sd_journal_get_data()</function>, but writes the result data to the file descriptor specified
+    by the <parameter>fd</parameter> argument. If the <parameter>ret_size</parameter> argument is
+    non-<constant>NULL</constant>, the number of bytes written to the file descriptor is stored there. Data
+    is written starting at the <parameter>fd</parameter>'s current offset, and the <parameter>fd</parameter>
+    is never rewound. On failure, the <parameter>fd</parameter> may already contain partially written data.
+    </para>
+
     <para><function>sd_journal_enumerate_data()</function> may be used
     to iterate through all fields of the current entry. On each
     invocation the data for the next field is returned. The order of
@@ -137,6 +156,22 @@
 
     <para><function>sd_journal_get_data_threshold()</function> returns
     the currently configured data field size threshold.</para>
+
+    <para>The <parameter>flags</parameter> argument is a bitmask of zero or more of the following
+    flags:</para>
+
+    <variablelist>
+      <varlistentry>
+        <term><constant>SD_JOURNAL_DATA_SKIP_FIELD</constant></term>
+
+        <listitem>
+          <para>By default, the result data is prefixed with the field name and <literal>=</literal>,
+          matching the format returned by <function>sd_journal_get_data()</function>. When this flag is
+          specified, the prefix is omitted and only the field's value is written to the file descriptor.
+          </para>
+        </listitem>
+      </varlistentry>
+    </variablelist>
   </refsect1>
 
   <refsect1>
@@ -246,6 +281,14 @@
 
           <xi:include href="version-info.xml" xpointer="v246"/></listitem>
         </varlistentry>
+
+        <varlistentry id='EBADF'>
+          <term><constant>-EBADF</constant></term>
+
+          <listitem><para>An invalid file descriptor is passed to
+          <function>sd_journal_get_data_to_fd()</function>.</para></listitem>
+
+        </varlistentry>
       </variablelist>
     </refsect2>
   </refsect1>
@@ -290,6 +333,7 @@ int print_fields(sd_journal *j) {
     <para><function>sd_journal_set_data_threshold()</function> and
     <function>sd_journal_get_data_threshold()</function> were added in version 196.</para>
     <para><function>sd_journal_enumerate_available_data()</function> was added in version 246.</para>
+    <para><function>sd_journal_get_data_to_fd()</function> was added in version 263.</para>
   </refsect1>
 
   <refsect1>

src/libsystemd/forward.h

@@ -118,6 +118,8 @@ typedef struct sd_varlink_interface sd_varlink_interface;
 typedef int (*sd_varlink_method_t)(sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata);
 
 typedef struct sd_journal sd_journal;
+enum ENUM_TYPE_S64(sd_journal_data_flags_t);
+typedef enum sd_journal_data_flags_t sd_journal_data_flags_t;
 
 typedef struct sd_resolve sd_resolve;
 typedef struct sd_resolve_query sd_resolve_query;

src/libsystemd/libsystemd.sym

@@ -1117,6 +1117,7 @@ LIBSYSTEMD_263 {
 global:
         sd_device_new_from_device_node_fd;
         sd_device_trigger_with_args;
+        sd_journal_get_data_to_fd;
         sd_varlink_invoke_and_upgrade;
         sd_varlink_invoke_and_upgradeb;
 } LIBSYSTEMD_262;

src/libsystemd/sd-journal/journal-file.c

@@ -11,6 +11,7 @@
 #include <unistd.h>
 
 #include "sd-event.h"
+#include "sd-journal.h"
 
 #include "alloc-util.h"
 #include "chattr-util.h"
@@ -2054,10 +2055,16 @@ static int maybe_decompress_payload(
                 Compression compression,
                 const char *field,
                 size_t field_length,
+                sd_journal_data_flags_t flags,
+                int fd,
                 const void **ret_data,
                 size_t *ret_size) {
 
+        int r;
+
         assert(f);
+        assert(!field == (field_length == 0)); /* These must be specified together. */
+        assert(fd < 0 || !ret_data); /* fd and ret_data cannot be specified together. */
 
         /* We can't read objects larger than 4G on a 32-bit machine */
 #if __SIZEOF_SIZE_T__ == 4
@@ -2065,10 +2072,11 @@ static int maybe_decompress_payload(
                 return -E2BIG;
 #endif
 
+        size_t skip = FLAGS_SET(flags, SD_JOURNAL_DATA_SKIP_FIELD) && field_length > 0 ? field_length + 1 : 0;
+
         if (compression != COMPRESSION_NONE) {
 #if HAVE_COMPRESSION
                 size_t rsize;
-                int r;
 
                 if (field) {
                         r = decompress_startswith_journal(
@@ -2088,18 +2096,31 @@ static int maybe_decompress_payload(
                         }
 
                         /* Caller only wants to check field existence, skip full decompression */
-                        if (!ret_data && !ret_size)
+                        if (fd < 0 && !ret_data && !ret_size)
                                 return 1;
                 }
 
-                r = decompress_blob_journal(compression, payload, size, &f->compress_buffer, &rsize, DATA_SIZE_MAX);
-                if (r < 0)
-                        return r;
+                if (fd >= 0) {
+                        r = decompress_blob_to_fd_journal(
+                                        compression, payload, size, fd,
+                                        DATA_SIZE_MAX, skip, ret_size);
+                        if (ERRNO_IS_NEG_JOURNAL_CORRUPTION(r))
+                                /* The caller journal_get_data() ignores these errors. Let's use a different
+                                 * error code in that case. Otherwise, junk data may be prepended to the
+                                 * beginning even on success. */
+                                return -EIO;
+                        if (r < 0)
+                                return r;
+                } else {
+                        r = decompress_blob_journal(compression, payload, size, &f->compress_buffer, &rsize, DATA_SIZE_MAX);
+                        if (r < 0)
+                                return r;
 
-                if (ret_data)
-                        *ret_data = f->compress_buffer;
-                if (ret_size)
-                        *ret_size = rsize;
+                        if (ret_data)
+                                *ret_data = (uint8_t*) f->compress_buffer + skip;
+                        if (ret_size)
+                                *ret_size = rsize - skip;
+                }
 #else
                 return -EPROTONOSUPPORT;
 #endif
@@ -2112,6 +2133,15 @@ static int maybe_decompress_payload(
                         return 0;
                 }
 
+                payload += skip;
+                size -= skip;
+
+                if (fd >= 0) {
+                        r = loop_write(fd, payload, size);
+                        if (r < 0)
+                                return r;
+                }
+
                 if (ret_data)
                         *ret_data = payload;
                 if (ret_size)
@@ -2121,12 +2151,14 @@ static int maybe_decompress_payload(
         return 1;
 }
 
-int journal_file_data_payload(
+int journal_file_data_payload_full(
                 JournalFile *f,
                 Object *o,
                 uint64_t offset,
                 const char *field,
                 size_t field_length,
+                sd_journal_data_flags_t flags,
+                int fd,
                 const void **ret_data,
                 size_t *ret_size) {
 
@@ -2136,6 +2168,7 @@ int journal_file_data_payload(
 
         assert(f);
         assert(!field == (field_length == 0)); /* These must be specified together. */
+        assert(fd < 0 || !ret_data); /* fd and ret_data cannot be specified together. */
 
         if (!o) {
                 r = journal_file_move_to_object(f, OBJECT_DATA, offset, &o);
@@ -2155,7 +2188,7 @@ int journal_file_data_payload(
 
         return maybe_decompress_payload(
                         f, journal_file_data_payload_field(f, o), size, c, field, field_length,
-                        ret_data, ret_size);
+                        flags, fd, ret_data, ret_size);
 }
 
 int journal_file_data_payload_pinned(
@@ -2213,13 +2246,15 @@ int journal_file_entry_n_fields(JournalFile *f, Object *o, uint64_t offset, uint
         return 0;
 }
 
-int journal_file_entry_field_payload(
+int journal_file_entry_field_payload_full(
                 JournalFile *f,
                 Object *o,
                 uint64_t offset,
                 uint64_t i,
                 const char *field,
                 size_t field_length,
+                sd_journal_data_flags_t flags,
+                int fd,
                 const void **ret_data,
                 size_t *ret_size) {
 
@@ -2235,11 +2270,12 @@ int journal_file_entry_field_payload(
         if (i >= n)
                 return -EADDRNOTAVAIL;
 
-        return journal_file_data_payload(
+        return journal_file_data_payload_full(
                         f,
                         /* o= */ NULL,
                         journal_file_entry_item_object_offset(f, o, i),
                         field, field_length,
+                        flags, fd,
                         ret_data, ret_size);
 }
 

src/libsystemd/sd-journal/journal-file.h

@@ -4,6 +4,8 @@
 #include <fcntl.h>
 #include <sys/stat.h>
 
+#include "sd-journal.h"
+
 #include "compress.h"
 #include "forward.h"
 #include "journal-def.h"
@@ -229,14 +231,32 @@ static inline size_t journal_file_entry_item_size(JournalFile *f) {
 
 uint64_t journal_file_entry_n_items(JournalFile *f, Object *o) _pure_;
 
-int journal_file_data_payload(
+int journal_file_data_payload_full(
+                JournalFile *f,
+                Object *o,
+                uint64_t offset,
+                const char *field,
+                size_t field_length,
+                sd_journal_data_flags_t flags,
+                int fd,
+                const void **ret_data,
+                size_t *ret_size);
+
+static inline int journal_file_data_payload(
                 JournalFile *f,
                 Object *o,
                 uint64_t offset,
                 const char *field,
                 size_t field_length,
                 const void **ret_data,
-                size_t *ret_size);
+                size_t *ret_size) {
+
+        return journal_file_data_payload_full(
+                        f, o, offset, field, field_length,
+                        /* flags= */ 0, /* fd= */ -EBADF,
+                        ret_data, ret_size);
+}
+
 int journal_file_data_payload_pinned(
                 JournalFile *f,
                 uint64_t offset,
@@ -257,7 +277,19 @@ static inline uint8_t* journal_file_data_payload_field(JournalFile *f, Object *o
 
 /* The fields of an entry. Classic files ignore the offset of the entry object. */
 int journal_file_entry_n_fields(JournalFile *f, Object *o, uint64_t offset, uint64_t *ret);
-int journal_file_entry_field_payload(
+int journal_file_entry_field_payload_full(
+                JournalFile *f,
+                Object *o,
+                uint64_t offset,
+                uint64_t i,
+                const char *field,
+                size_t field_length,
+                sd_journal_data_flags_t flags,
+                int fd,
+                const void **ret_data,
+                size_t *ret_size);
+
+static inline int journal_file_entry_field_payload(
                 JournalFile *f,
                 Object *o,
                 uint64_t offset,
@@ -265,7 +297,13 @@ int journal_file_entry_field_payload(
                 const char *field,
                 size_t field_length,
                 const void **ret_data,
-                size_t *ret_size);
+                size_t *ret_size) {
+
+        return journal_file_entry_field_payload_full(
+                        f, o, offset, i, field, field_length,
+                        /* flags= */ 0, /* fd= */ -EBADF,
+                        ret_data, ret_size);
+}
 
 uint64_t journal_file_entry_array_n_items(JournalFile *f, Object *o) _pure_;
 

src/libsystemd/sd-journal/sd-journal.c

@@ -2926,7 +2926,14 @@ static bool field_is_valid(const char *field) {
         return true;
 }
 
-_public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **ret_data, size_t *ret_size) {
+static int journal_get_data(
+                sd_journal *j,
+                const char *field,
+                sd_journal_data_flags_t flags,
+                int fd,
+                const void **ret_data,
+                size_t *ret_size) {
+
         JournalFile *f;
         size_t field_length;
         Object *o;
@@ -2959,9 +2966,9 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **
                 const void *d;
                 size_t l;
 
-                r = journal_file_entry_field_payload(
+                r = journal_file_entry_field_payload_full(
                                 f, o, f->current_offset, i, field, field_length,
-                                ret_data ? &d : NULL, ret_size ? &l : NULL);
+                                flags, fd, ret_data ? &d : NULL, ret_size ? &l : NULL);
                 if (r == 0)
                         continue;
                 if (ERRNO_IS_NEG_JOURNAL_CORRUPTION(r)) {
@@ -2982,6 +2989,17 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **
         return -ENOENT;
 }
 
+_public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **ret_data, size_t *ret_size) {
+        return journal_get_data(j, field, /* flags= */ 0, /* fd= */ -EBADF, ret_data, ret_size);
+}
+
+_public_ int sd_journal_get_data_to_fd(sd_journal *j, const char *field, sd_journal_data_flags_t flags, int fd, size_t *ret_size) {
+        assert_return(fd >= 0, -EBADF);
+        assert_return((flags & ~SD_JOURNAL_DATA_SKIP_FIELD) == 0, -EINVAL);
+
+        return journal_get_data(j, field, flags, fd, /* ret_data= */ NULL, ret_size);
+}
+
 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **ret_data, size_t *ret_size) {
         JournalFile *f;
         Object *o;

src/libsystemd/sd-journal/test-journal.c

@@ -7,12 +7,14 @@
 
 #include "argv-util.h"
 #include "chattr-util.h"
+#include "fd-util.h"
 #include "iovec-util.h"
 #include "journal-authenticate.h"
 #include "journal-file-util.h"
 #include "journal-vacuum.h"
 #include "log.h"
 #include "lookup3.h"
+#include "memfd-util.h"
 #include "memory-util.h"
 #include "rm-rf.h"
 #include "stdio-util.h"
@@ -253,14 +255,21 @@ static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) {
         _cleanup_(mmap_cache_unrefp) MMapCache *m = NULL;
         dual_timestamp ts;
         JournalFile *f;
-        struct iovec iovec;
         Object *o;
         uint64_t p;
         char t[] = "/var/tmp/journal-XXXXXX";
-        char data[2048] = "FIELD=";
+        const char *field = "FIELD";
         bool is_compressed;
 
-        ASSERT_LE(data_size, sizeof(data));
+        size_t field_length = strlen(field);
+        ASSERT_GT(data_size, field_length);
+        ASSERT_LE(data_size, (uint64_t) SIZE_MAX);
+
+        _cleanup_(iovec_done) struct iovec iovec = {};
+        ASSERT_OK(iovec_alloc(data_size, &iovec));
+        char *q = mempcpy(iovec.iov_base, field, field_length);
+        *q++ = '=';
+        memset(q, 'a', data_size - field_length - 1);
 
         ASSERT_NOT_NULL(m = mmap_cache_new());
 
@@ -270,7 +279,6 @@ static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) {
 
         dual_timestamp_now(&ts);
 
-        iovec = IOVEC_MAKE(data, data_size);
         ASSERT_OK_ZERO(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL, NULL));
 
         journal_file_auth_append_tag(f);
@@ -291,6 +299,32 @@ static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) {
 
         is_compressed = COMPRESSION_FROM_OBJECT(o) != COMPRESSION_NONE;
 
+        sd_journal_data_flags_t flags;
+        FOREACH_ARGUMENT(flags, 0, SD_JOURNAL_DATA_SKIP_FIELD) {
+                size_t skip = FLAGS_SET(flags, SD_JOURNAL_DATA_SKIP_FIELD) ? field_length + 1 : 0;
+
+                struct iovec v;
+                ASSERT_OK_POSITIVE(journal_file_data_payload_full(f, o, p, field, field_length, flags,
+                                                                  -EBADF, (const void**) &v.iov_base, &v.iov_len));
+                ASSERT_TRUE(iovec_equal(&v, &IOVEC_SHIFT(&iovec, skip)));
+
+                _cleanup_close_ int fd = ASSERT_OK(memfd_new("decompress-blob-to-fd-test"));
+                size_t sz;
+                ASSERT_OK_POSITIVE(journal_file_data_payload_full(f, o, p, field, field_length, flags,
+                                                                  fd, /* ret_data= */ NULL, &sz));
+                ASSERT_EQ((uint64_t) sz, data_size - skip);
+
+                uint64_t sz64;
+                ASSERT_OK(memfd_get_size(fd, &sz64));
+                ASSERT_EQ(sz64, data_size - skip);
+
+                _cleanup_(iovec_done) struct iovec w = {};
+                ASSERT_OK(iovec_alloc(sz, &w));
+
+                ASSERT_OK_EQ_ERRNO(pread(fd, w.iov_base, w.iov_len, 0), (ssize_t) w.iov_len);
+                ASSERT_TRUE(iovec_equal(&w, &IOVEC_SHIFT(&iovec, skip)));
+        }
+
         (void) journal_file_offline_close(f);
 
         log_info("Done...");

src/systemd/sd-journal.h

@@ -81,6 +81,12 @@ enum {
         SD_JOURNAL_INVALIDATE
 };
 
+/* Flags for reading data */
+__extension__ typedef enum _SD_ENUM_TYPE_S64(sd_journal_data_flags_t) {
+        SD_JOURNAL_DATA_SKIP_FIELD = 1 << 0,
+        _SD_ENUM_FORCE_S64(JOURNAL_DATA_FLAGS)
+} sd_journal_data_flags_t;
+
 int sd_journal_open(sd_journal **ret, int flags);
 int sd_journal_open_namespace(sd_journal **ret, const char *name_space, int flags);
 int sd_journal_open_directory(sd_journal **ret, const char *path, int flags);
@@ -105,6 +111,7 @@ int sd_journal_set_data_threshold(sd_journal *j, size_t sz);
 int sd_journal_get_data_threshold(sd_journal *j, size_t *sz);
 
 int sd_journal_get_data(sd_journal *j, const char *field, const void **ret_data, size_t *ret_size);
+int sd_journal_get_data_to_fd(sd_journal *j, const char *field, sd_journal_data_flags_t flags, int fd, size_t *ret_size);
 int sd_journal_enumerate_data(sd_journal *j, const void **ret_data, size_t *ret_size);
 int sd_journal_enumerate_available_data(sd_journal *j, const void **ret_data, size_t *ret_size);
 void sd_journal_restart_data(sd_journal *j);