From e007f2f0a4bc54be2b2d348a0d1657a993f66ca8 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Sun, 23 Aug 2026 09:50:24 +0900 Subject: [PATCH] sd-journal: introduce sd_journal_get_data_to_fd() It may be useful for getting large data, e.g. coredump, from journal. --- man/rules/meson.build | 4 +- man/sd_journal_get_data.xml | 44 +++++++++++++++++ src/libsystemd/forward.h | 2 + src/libsystemd/libsystemd.sym | 1 + src/libsystemd/sd-journal/journal-file.c | 62 +++++++++++++++++++----- src/libsystemd/sd-journal/journal-file.h | 46 ++++++++++++++++-- src/libsystemd/sd-journal/sd-journal.c | 24 +++++++-- src/libsystemd/sd-journal/test-journal.c | 42 ++++++++++++++-- src/systemd/sd-journal.h | 7 +++ 9 files changed, 207 insertions(+), 25 deletions(-) diff --git a/man/rules/meson.build b/man/rules/meson.build index 68aeb749392..c81f98bdf63 100644 --- a/man/rules/meson.build +++ b/man/rules/meson.build @@ -772,10 +772,12 @@ manpages = [ ''], ['sd_journal_get_data', '3', - ['SD_JOURNAL_FOREACH_DATA', + ['SD_JOURNAL_DATA_SKIP_FIELD', + 'SD_JOURNAL_FOREACH_DATA', 'sd_journal_enumerate_available_data', 'sd_journal_enumerate_data', 'sd_journal_get_data_threshold', + 'sd_journal_get_data_to_fd', 'sd_journal_restart_data', 'sd_journal_set_data_threshold'], ''], diff --git a/man/sd_journal_get_data.xml b/man/sd_journal_get_data.xml index 437d7c03b7e..65c2159ce07 100644 --- a/man/sd_journal_get_data.xml +++ b/man/sd_journal_get_data.xml @@ -17,12 +17,14 @@ sd_journal_get_data + sd_journal_get_data_to_fd sd_journal_enumerate_data sd_journal_enumerate_available_data sd_journal_restart_data SD_JOURNAL_FOREACH_DATA sd_journal_set_data_threshold sd_journal_get_data_threshold + SD_JOURNAL_DATA_SKIP_FIELD Read data fields from the current journal entry @@ -38,6 +40,15 @@ size_t *length + + int sd_journal_get_data_to_fd + sd_journal *j + const char *field + sd_journal_data_flags_t flags + int fd + size_t *ret_size + + int sd_journal_enumerate_data sd_journal *j @@ -94,6 +105,14 @@ sd_journal_enumerate_available_data(), or when the read pointer is altered. Note that the data returned will be prefixed with the field name and =. + sd_journal_get_data_to_fd() is similar to + sd_journal_get_data(), but writes the result data to the file descriptor specified + by the fd argument. If the ret_size argument is + non-NULL, the number of bytes written to the file descriptor is stored there. Data + is written starting at the fd's current offset, and the fd + is never rewound. On failure, the fd may already contain partially written data. + + sd_journal_enumerate_data() may be used to iterate through all fields of the current entry. On each invocation the data for the next field is returned. The order of @@ -137,6 +156,22 @@ sd_journal_get_data_threshold() returns the currently configured data field size threshold. + + The flags argument is a bitmask of zero or more of the following + flags: + + + + SD_JOURNAL_DATA_SKIP_FIELD + + + By default, the result data is prefixed with the field name and =, + matching the format returned by sd_journal_get_data(). When this flag is + specified, the prefix is omitted and only the field's value is written to the file descriptor. + + + + @@ -246,6 +281,14 @@ + + + -EBADF + + An invalid file descriptor is passed to + sd_journal_get_data_to_fd(). + + @@ -290,6 +333,7 @@ int print_fields(sd_journal *j) { sd_journal_set_data_threshold() and sd_journal_get_data_threshold() were added in version 196. sd_journal_enumerate_available_data() was added in version 246. + sd_journal_get_data_to_fd() was added in version 263. diff --git a/src/libsystemd/forward.h b/src/libsystemd/forward.h index 4206993924a..dabbfacf064 100644 --- a/src/libsystemd/forward.h +++ b/src/libsystemd/forward.h @@ -118,6 +118,8 @@ typedef struct sd_varlink_interface sd_varlink_interface; typedef int (*sd_varlink_method_t)(sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata); typedef struct sd_journal sd_journal; +enum ENUM_TYPE_S64(sd_journal_data_flags_t); +typedef enum sd_journal_data_flags_t sd_journal_data_flags_t; typedef struct sd_resolve sd_resolve; typedef struct sd_resolve_query sd_resolve_query; diff --git a/src/libsystemd/libsystemd.sym b/src/libsystemd/libsystemd.sym index aea64f65673..32ded0c495e 100644 --- a/src/libsystemd/libsystemd.sym +++ b/src/libsystemd/libsystemd.sym @@ -1117,6 +1117,7 @@ LIBSYSTEMD_263 { global: sd_device_new_from_device_node_fd; sd_device_trigger_with_args; + sd_journal_get_data_to_fd; sd_varlink_invoke_and_upgrade; sd_varlink_invoke_and_upgradeb; } LIBSYSTEMD_262; diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index be0ad02caf0..f990a10d5d6 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -11,6 +11,7 @@ #include #include "sd-event.h" +#include "sd-journal.h" #include "alloc-util.h" #include "chattr-util.h" @@ -2054,10 +2055,16 @@ static int maybe_decompress_payload( Compression compression, const char *field, size_t field_length, + sd_journal_data_flags_t flags, + int fd, const void **ret_data, size_t *ret_size) { + int r; + assert(f); + assert(!field == (field_length == 0)); /* These must be specified together. */ + assert(fd < 0 || !ret_data); /* fd and ret_data cannot be specified together. */ /* We can't read objects larger than 4G on a 32-bit machine */ #if __SIZEOF_SIZE_T__ == 4 @@ -2065,10 +2072,11 @@ static int maybe_decompress_payload( return -E2BIG; #endif + size_t skip = FLAGS_SET(flags, SD_JOURNAL_DATA_SKIP_FIELD) && field_length > 0 ? field_length + 1 : 0; + if (compression != COMPRESSION_NONE) { #if HAVE_COMPRESSION size_t rsize; - int r; if (field) { r = decompress_startswith_journal( @@ -2088,18 +2096,31 @@ static int maybe_decompress_payload( } /* Caller only wants to check field existence, skip full decompression */ - if (!ret_data && !ret_size) + if (fd < 0 && !ret_data && !ret_size) return 1; } - r = decompress_blob_journal(compression, payload, size, &f->compress_buffer, &rsize, DATA_SIZE_MAX); - if (r < 0) - return r; + if (fd >= 0) { + r = decompress_blob_to_fd_journal( + compression, payload, size, fd, + DATA_SIZE_MAX, skip, ret_size); + if (ERRNO_IS_NEG_JOURNAL_CORRUPTION(r)) + /* The caller journal_get_data() ignores these errors. Let's use a different + * error code in that case. Otherwise, junk data may be prepended to the + * beginning even on success. */ + return -EIO; + if (r < 0) + return r; + } else { + r = decompress_blob_journal(compression, payload, size, &f->compress_buffer, &rsize, DATA_SIZE_MAX); + if (r < 0) + return r; - if (ret_data) - *ret_data = f->compress_buffer; - if (ret_size) - *ret_size = rsize; + if (ret_data) + *ret_data = (uint8_t*) f->compress_buffer + skip; + if (ret_size) + *ret_size = rsize - skip; + } #else return -EPROTONOSUPPORT; #endif @@ -2112,6 +2133,15 @@ static int maybe_decompress_payload( return 0; } + payload += skip; + size -= skip; + + if (fd >= 0) { + r = loop_write(fd, payload, size); + if (r < 0) + return r; + } + if (ret_data) *ret_data = payload; if (ret_size) @@ -2121,12 +2151,14 @@ static int maybe_decompress_payload( return 1; } -int journal_file_data_payload( +int journal_file_data_payload_full( JournalFile *f, Object *o, uint64_t offset, const char *field, size_t field_length, + sd_journal_data_flags_t flags, + int fd, const void **ret_data, size_t *ret_size) { @@ -2136,6 +2168,7 @@ int journal_file_data_payload( assert(f); assert(!field == (field_length == 0)); /* These must be specified together. */ + assert(fd < 0 || !ret_data); /* fd and ret_data cannot be specified together. */ if (!o) { r = journal_file_move_to_object(f, OBJECT_DATA, offset, &o); @@ -2155,7 +2188,7 @@ int journal_file_data_payload( return maybe_decompress_payload( f, journal_file_data_payload_field(f, o), size, c, field, field_length, - ret_data, ret_size); + flags, fd, ret_data, ret_size); } int journal_file_data_payload_pinned( @@ -2213,13 +2246,15 @@ int journal_file_entry_n_fields(JournalFile *f, Object *o, uint64_t offset, uint return 0; } -int journal_file_entry_field_payload( +int journal_file_entry_field_payload_full( JournalFile *f, Object *o, uint64_t offset, uint64_t i, const char *field, size_t field_length, + sd_journal_data_flags_t flags, + int fd, const void **ret_data, size_t *ret_size) { @@ -2235,11 +2270,12 @@ int journal_file_entry_field_payload( if (i >= n) return -EADDRNOTAVAIL; - return journal_file_data_payload( + return journal_file_data_payload_full( f, /* o= */ NULL, journal_file_entry_item_object_offset(f, o, i), field, field_length, + flags, fd, ret_data, ret_size); } diff --git a/src/libsystemd/sd-journal/journal-file.h b/src/libsystemd/sd-journal/journal-file.h index 7593dd264e0..9bfbf61b14b 100644 --- a/src/libsystemd/sd-journal/journal-file.h +++ b/src/libsystemd/sd-journal/journal-file.h @@ -4,6 +4,8 @@ #include #include +#include "sd-journal.h" + #include "compress.h" #include "forward.h" #include "journal-def.h" @@ -229,14 +231,32 @@ static inline size_t journal_file_entry_item_size(JournalFile *f) { uint64_t journal_file_entry_n_items(JournalFile *f, Object *o) _pure_; -int journal_file_data_payload( +int journal_file_data_payload_full( + JournalFile *f, + Object *o, + uint64_t offset, + const char *field, + size_t field_length, + sd_journal_data_flags_t flags, + int fd, + const void **ret_data, + size_t *ret_size); + +static inline int journal_file_data_payload( JournalFile *f, Object *o, uint64_t offset, const char *field, size_t field_length, const void **ret_data, - size_t *ret_size); + size_t *ret_size) { + + return journal_file_data_payload_full( + f, o, offset, field, field_length, + /* flags= */ 0, /* fd= */ -EBADF, + ret_data, ret_size); +} + int journal_file_data_payload_pinned( JournalFile *f, uint64_t offset, @@ -257,7 +277,19 @@ static inline uint8_t* journal_file_data_payload_field(JournalFile *f, Object *o /* The fields of an entry. Classic files ignore the offset of the entry object. */ int journal_file_entry_n_fields(JournalFile *f, Object *o, uint64_t offset, uint64_t *ret); -int journal_file_entry_field_payload( +int journal_file_entry_field_payload_full( + JournalFile *f, + Object *o, + uint64_t offset, + uint64_t i, + const char *field, + size_t field_length, + sd_journal_data_flags_t flags, + int fd, + const void **ret_data, + size_t *ret_size); + +static inline int journal_file_entry_field_payload( JournalFile *f, Object *o, uint64_t offset, @@ -265,7 +297,13 @@ int journal_file_entry_field_payload( const char *field, size_t field_length, const void **ret_data, - size_t *ret_size); + size_t *ret_size) { + + return journal_file_entry_field_payload_full( + f, o, offset, i, field, field_length, + /* flags= */ 0, /* fd= */ -EBADF, + ret_data, ret_size); +} uint64_t journal_file_entry_array_n_items(JournalFile *f, Object *o) _pure_; diff --git a/src/libsystemd/sd-journal/sd-journal.c b/src/libsystemd/sd-journal/sd-journal.c index b3a688d58b7..ab66602e52c 100644 --- a/src/libsystemd/sd-journal/sd-journal.c +++ b/src/libsystemd/sd-journal/sd-journal.c @@ -2926,7 +2926,14 @@ static bool field_is_valid(const char *field) { return true; } -_public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **ret_data, size_t *ret_size) { +static int journal_get_data( + sd_journal *j, + const char *field, + sd_journal_data_flags_t flags, + int fd, + const void **ret_data, + size_t *ret_size) { + JournalFile *f; size_t field_length; Object *o; @@ -2959,9 +2966,9 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void ** const void *d; size_t l; - r = journal_file_entry_field_payload( + r = journal_file_entry_field_payload_full( f, o, f->current_offset, i, field, field_length, - ret_data ? &d : NULL, ret_size ? &l : NULL); + flags, fd, ret_data ? &d : NULL, ret_size ? &l : NULL); if (r == 0) continue; if (ERRNO_IS_NEG_JOURNAL_CORRUPTION(r)) { @@ -2982,6 +2989,17 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void ** return -ENOENT; } +_public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **ret_data, size_t *ret_size) { + return journal_get_data(j, field, /* flags= */ 0, /* fd= */ -EBADF, ret_data, ret_size); +} + +_public_ int sd_journal_get_data_to_fd(sd_journal *j, const char *field, sd_journal_data_flags_t flags, int fd, size_t *ret_size) { + assert_return(fd >= 0, -EBADF); + assert_return((flags & ~SD_JOURNAL_DATA_SKIP_FIELD) == 0, -EINVAL); + + return journal_get_data(j, field, flags, fd, /* ret_data= */ NULL, ret_size); +} + _public_ int sd_journal_enumerate_data(sd_journal *j, const void **ret_data, size_t *ret_size) { JournalFile *f; Object *o; diff --git a/src/libsystemd/sd-journal/test-journal.c b/src/libsystemd/sd-journal/test-journal.c index b5ef8824b7e..fb41c40db64 100644 --- a/src/libsystemd/sd-journal/test-journal.c +++ b/src/libsystemd/sd-journal/test-journal.c @@ -7,12 +7,14 @@ #include "argv-util.h" #include "chattr-util.h" +#include "fd-util.h" #include "iovec-util.h" #include "journal-authenticate.h" #include "journal-file-util.h" #include "journal-vacuum.h" #include "log.h" #include "lookup3.h" +#include "memfd-util.h" #include "memory-util.h" #include "rm-rf.h" #include "stdio-util.h" @@ -253,14 +255,21 @@ static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) { _cleanup_(mmap_cache_unrefp) MMapCache *m = NULL; dual_timestamp ts; JournalFile *f; - struct iovec iovec; Object *o; uint64_t p; char t[] = "/var/tmp/journal-XXXXXX"; - char data[2048] = "FIELD="; + const char *field = "FIELD"; bool is_compressed; - ASSERT_LE(data_size, sizeof(data)); + size_t field_length = strlen(field); + ASSERT_GT(data_size, field_length); + ASSERT_LE(data_size, (uint64_t) SIZE_MAX); + + _cleanup_(iovec_done) struct iovec iovec = {}; + ASSERT_OK(iovec_alloc(data_size, &iovec)); + char *q = mempcpy(iovec.iov_base, field, field_length); + *q++ = '='; + memset(q, 'a', data_size - field_length - 1); ASSERT_NOT_NULL(m = mmap_cache_new()); @@ -270,7 +279,6 @@ static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) { dual_timestamp_now(&ts); - iovec = IOVEC_MAKE(data, data_size); ASSERT_OK_ZERO(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL, NULL)); journal_file_auth_append_tag(f); @@ -291,6 +299,32 @@ static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) { is_compressed = COMPRESSION_FROM_OBJECT(o) != COMPRESSION_NONE; + sd_journal_data_flags_t flags; + FOREACH_ARGUMENT(flags, 0, SD_JOURNAL_DATA_SKIP_FIELD) { + size_t skip = FLAGS_SET(flags, SD_JOURNAL_DATA_SKIP_FIELD) ? field_length + 1 : 0; + + struct iovec v; + ASSERT_OK_POSITIVE(journal_file_data_payload_full(f, o, p, field, field_length, flags, + -EBADF, (const void**) &v.iov_base, &v.iov_len)); + ASSERT_TRUE(iovec_equal(&v, &IOVEC_SHIFT(&iovec, skip))); + + _cleanup_close_ int fd = ASSERT_OK(memfd_new("decompress-blob-to-fd-test")); + size_t sz; + ASSERT_OK_POSITIVE(journal_file_data_payload_full(f, o, p, field, field_length, flags, + fd, /* ret_data= */ NULL, &sz)); + ASSERT_EQ((uint64_t) sz, data_size - skip); + + uint64_t sz64; + ASSERT_OK(memfd_get_size(fd, &sz64)); + ASSERT_EQ(sz64, data_size - skip); + + _cleanup_(iovec_done) struct iovec w = {}; + ASSERT_OK(iovec_alloc(sz, &w)); + + ASSERT_OK_EQ_ERRNO(pread(fd, w.iov_base, w.iov_len, 0), (ssize_t) w.iov_len); + ASSERT_TRUE(iovec_equal(&w, &IOVEC_SHIFT(&iovec, skip))); + } + (void) journal_file_offline_close(f); log_info("Done..."); diff --git a/src/systemd/sd-journal.h b/src/systemd/sd-journal.h index 11314f1fb2d..d34424499b6 100644 --- a/src/systemd/sd-journal.h +++ b/src/systemd/sd-journal.h @@ -81,6 +81,12 @@ enum { SD_JOURNAL_INVALIDATE }; +/* Flags for reading data */ +__extension__ typedef enum _SD_ENUM_TYPE_S64(sd_journal_data_flags_t) { + SD_JOURNAL_DATA_SKIP_FIELD = 1 << 0, + _SD_ENUM_FORCE_S64(JOURNAL_DATA_FLAGS) +} sd_journal_data_flags_t; + int sd_journal_open(sd_journal **ret, int flags); int sd_journal_open_namespace(sd_journal **ret, const char *name_space, int flags); int sd_journal_open_directory(sd_journal **ret, const char *path, int flags); @@ -105,6 +111,7 @@ int sd_journal_set_data_threshold(sd_journal *j, size_t sz); int sd_journal_get_data_threshold(sd_journal *j, size_t *sz); int sd_journal_get_data(sd_journal *j, const char *field, const void **ret_data, size_t *ret_size); +int sd_journal_get_data_to_fd(sd_journal *j, const char *field, sd_journal_data_flags_t flags, int fd, size_t *ret_size); int sd_journal_enumerate_data(sd_journal *j, const void **ret_data, size_t *ret_size); int sd_journal_enumerate_available_data(sd_journal *j, const void **ret_data, size_t *ret_size); void sd_journal_restart_data(sd_journal *j);