sysupdate: support UAPI.16 manifests as alternative to SHA256SUMS

man/sysupdate.d.xml

@@ -140,7 +140,10 @@
       <listitem><para>For HTTP/HTTPS sources a simple scheme is used: a manifest file
       <filename>SHA256SUMS</filename>, following the format defined by <citerefentry
       project='man-pages'><refentrytitle>sha256sum</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
-      lists file names and their SHA256 hashes.</para></listitem>
+      lists file names and their SHA256 hashes. Alternatively, a <filename>Uapi16Manifest</filename> file
+      following the <ulink url="https://uapi-group.org/specifications/specs/file_manifest/">UAPI.16 File
+      Manifest</ulink> specification may be used, see <varname>ManifestFormat=</varname>
+      below.</para></listitem>
     </itemizedlist>
 
     <para>Transfers are done in the alphabetical order of the <filename>.transfer</filename> file names they are
@@ -530,7 +533,9 @@
 
         <para>This option only has an effect if the source resource type is selected as
         <constant>url-file</constant> or <constant>url-tar</constant>, as integrity and authentication
-        checking is only available for transfers from remote sources.</para>
+        checking is only available for transfers from remote sources. It currently also only has an effect if
+        <varname>ManifestFormat=sha256sums</varname> is used, as signatures of UAPI.16 manifests are not
+        verified yet.</para>
 
         <xi:include href="version-info.xml" xpointer="v251"/></listitem>
       </varlistentry>
@@ -634,11 +639,12 @@
         <listitem><para>Specifies where to find source versions of this resource.</para>
 
         <para>If the source type is selected as <constant>url-file</constant> or
-        <constant>url-tar</constant> this must be a HTTP/HTTPS URL. The URL is suffixed with
-        <filename>/SHA256SUMS</filename> to acquire the manifest file, with
-        <filename>/SHA256SUMS.gpg</filename> to acquire the detached signature file for it, and with the file
-        names listed in the manifest file in case an update is executed and a resource shall be
-        downloaded.</para>
+        <constant>url-tar</constant> this must be a HTTP/HTTPS URL. Unless <varname>ManifestPath=</varname>
+        is specified, the URL is suffixed with <filename>/SHA256SUMS</filename> (or
+        <filename>/Uapi16Manifest</filename>, depending on <varname>ManifestFormat=</varname>) to acquire the
+        manifest file, and with <filename>/SHA256SUMS.gpg</filename> to acquire the detached signature file
+        for it. It is suffixed with the file names listed in the manifest file in case an update is executed
+        and a resource shall be downloaded.</para>
 
         <para>Alternatively, for these two source types a URL of the form
         <literal>provider:[<replaceable>socket</replaceable>]/<replaceable>resource</replaceable></literal>
@@ -698,8 +704,17 @@
         <constant>url-file</constant> or <constant>url-tar</constant> source. Takes
         <constant>sha256sums</constant>, which selects a <filename>SHA256SUMS</filename> file as generated by
         <citerefentry project='man-pages'><refentrytitle>sha256sum</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
-        accompanied by a detached <filename>SHA256SUMS.gpg</filename> signature, see above. Defaults to
-        <constant>sha256sums</constant>. This option has no effect for other source resource types.</para>
+        accompanied by a detached <filename>SHA256SUMS.gpg</filename> signature, see above. Or
+        <constant>uapi16</constant>, which selects a <filename>Uapi16Manifest</filename> file following the
+        <ulink url="https://uapi-group.org/specifications/specs/file_manifest/">UAPI.16 File
+        Manifest</ulink> specification, i.e. a JSON-SEQ stream with one object per file, carrying the file
+        name, size and SHA256 digest of each file, as generated by
+        <citerefentry><refentrytitle>systemd-manifest</refentrytitle><manvolnum>1</manvolnum></citerefentry>
+        or <command>systemd-dissect --uapi16</command>. Only regular files listed in the manifest are
+        considered, and entries whose GPT partition type does not match the target's
+        <varname>MatchPartitionType=</varname> are ignored. Note that signature objects embedded in UAPI.16
+        manifests are currently not verified. Defaults to <constant>sha256sums</constant>. This option has no
+        effect for other source resource types.</para>
 
         <xi:include href="version-info.xml" xpointer="v263"/></listitem>
       </varlistentry>

src/shared/forward.h

@@ -95,6 +95,7 @@ typedef struct Table Table;
 typedef struct Tpm2Context Tpm2Context;
 typedef struct Tpm2Handle Tpm2Handle;
 typedef struct Tpm2PCRValue Tpm2PCRValue;
+typedef struct Uapi16Contents Uapi16Contents;
 typedef struct Uapi16Object Uapi16Object;
 typedef struct UnitInfo UnitInfo;
 typedef struct UnitResult UnitResult;

src/shared/varlink-io.systemd.SysUpdate.Provider.c

@@ -53,7 +53,9 @@ static SD_VARLINK_DEFINE_ENUM_TYPE(
 static SD_VARLINK_DEFINE_ENUM_TYPE(
                 ManifestFormat,
                 SD_VARLINK_FIELD_COMMENT("A SHA256SUMS file, see ManifestFormat=sha256sums in sysupdate.d(5)."),
-                SD_VARLINK_DEFINE_ENUM_VALUE(sha256sums));
+                SD_VARLINK_DEFINE_ENUM_VALUE(sha256sums),
+                SD_VARLINK_FIELD_COMMENT("A UAPI.16 manifest, see ManifestFormat=uapi16 in sysupdate.d(5)."),
+                SD_VARLINK_DEFINE_ENUM_VALUE(uapi16));
 
 static SD_VARLINK_DEFINE_STRUCT_TYPE(
                 TransferSource,

src/sysupdate/sysupdate-instance.c

@@ -3,6 +3,7 @@
 #include "alloc-util.h"
 #include "log.h"
 #include "sysupdate-instance.h"
+#include "uapi16-manifest.h"
 
 void instance_metadata_destroy(InstanceMetadata *m) {
         assert(m);

src/sysupdate/sysupdate-instance.h

@@ -8,8 +8,8 @@
 #include "sysupdate-partition.h"
 
 typedef struct InstanceMetadata {
-        /* Various bits of metadata for each instance, that is either derived from the filename/GPT label or
-         * from metadata of the file/partition itself */
+        /* Various bits of metadata for each instance, that is either derived from the filename/GPT label,
+         * from metadata of the file/partition itself or from the manifest */
         char *version;
         sd_id128_t partition_uuid;
         bool partition_uuid_set;
@@ -49,6 +49,11 @@ struct Instance {
         char *path;  /* includes the `.sysupdate.partial.` (etc.) prefix, if applicable */
         PartitionInfo partition_info;
 
+        /* The sources we can acquire the resource from (for web backed sources) */
+        Uapi16Contents *sources;
+        size_t n_sources;
+
+        /* The current state of this instance */
         bool is_partial;
         bool is_pending;
 };

src/sysupdate/sysupdate-resource.c

@@ -37,6 +37,7 @@
 #include "sysupdate-pattern.h"
 #include "sysupdate-resource.h"
 #include "time-util.h"
+#include "uapi16-manifest.h"
 
 void resource_destroy(Resource *rr) {
         assert(rr);
@@ -361,11 +362,22 @@ static int resource_load_from_blockdev(Resource *rr) {
         return 0;
 }
 
+static const char* const resource_manifest_filename_table[_MANIFEST_FORMAT_MAX] = {
+        [MANIFEST_FORMAT_SHA256SUMS] = "SHA256SUMS",
+        [MANIFEST_FORMAT_UAPI16]     = UAPI16_MANIFEST_FILENAME,
+};
+
+DEFINE_PRIVATE_STRING_TABLE_LOOKUP_TO_STRING(resource_manifest_filename, ManifestFormat);
+
+static bool resource_manifest_has_inline_signatures(ManifestFormat style) {
+        return style == MANIFEST_FORMAT_UAPI16;
+}
+
 static int resource_manifest_url(Resource *rr, char **ret) {
         int r;
 
         assert(rr);
-        assert(rr->manifest_format == MANIFEST_FORMAT_SHA256SUMS);
+        assert(IN_SET(rr->manifest_format, MANIFEST_FORMAT_SHA256SUMS, MANIFEST_FORMAT_UAPI16));
         assert(ret);
 
         /* Returns the URL to acquire the manifest from: either the one explicitly configured, or the
@@ -379,9 +391,10 @@ static int resource_manifest_url(Resource *rr, char **ret) {
                 return 0;
         }
 
-        r = import_url_append_component(rr->path, "SHA256SUMS", ret);
+        const char *fn = resource_manifest_filename_to_string(rr->manifest_format);
+        r = import_url_append_component(rr->path, fn, ret);
         if (r < 0)
-                return log_error_errno(r, "Failed to append SHA256SUMS to URL: %m");
+                return log_error_errno(r, "Failed to append '%s' to URL '%s': %m", fn, rr->path);
 
         return 0;
 }
@@ -394,11 +407,11 @@ static int download_manifest(
 
         int r;
 
-        assert(manifest_format == MANIFEST_FORMAT_SHA256SUMS);
+        assert(IN_SET(manifest_format, MANIFEST_FORMAT_SHA256SUMS, MANIFEST_FORMAT_UAPI16));
         assert(manifest_url);
         assert(ret);
 
-        /* Download a SHA256SUMS file as manifest */
+        /* Download a SHA256SUMS/Uapi16Manifest file as manifest */
 
         _cleanup_close_pair_ int pfd[2] = EBADF_PAIR;
         if (pipe2(pfd, O_CLOEXEC) < 0)
@@ -424,7 +437,7 @@ static int download_manifest(
                         "raw",
                         "--direct",                        /* just download the specified URL, don't download anything else */
                         "--checksum-file=yes",             /* whatever its name, look for <name>.gpg/.asc signature */
-                        "--verify", verify_signature ? "signature" : "no", /* verify the manifest file */
+                        "--verify", verify_signature && !resource_manifest_has_inline_signatures(manifest_format) ? "signature" : "no", /* verify the manifest file */
                         manifest_url,
                         "-",                               /* write to stdout */
                         NULL
@@ -626,10 +639,170 @@ static int sha256sums_process(
         }
 }
 
+static int instance_copy_uapi16(Instance *instance, const Uapi16Object *object) {
+        assert(instance);
+        assert(object);
+
+        /* We verified earlier that these fields are definitely set in the UAPI 16 objects */
+        assert(object->type == S_IFREG);
+        assert(iovec_is_set(&object->sha256));
+        assert(object->size != UINT64_MAX);
+
+        if (instance->metadata.sha256sum_set) {
+                if (memcmp_nn(instance->metadata.sha256sum, SHA256_DIGEST_SIZE, object->sha256.iov_base, object->sha256.iov_len) != 0)
+                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "SHA256 sum parsed from filename and manifest don't match, refusing.");
+        } else {
+                assert(object->sha256.iov_len == SHA256_DIGEST_SIZE);
+                memcpy(instance->metadata.sha256sum, object->sha256.iov_base, SHA256_DIGEST_SIZE);
+                instance->metadata.sha256sum_set = true;
+        }
+
+        if (instance->metadata.size != UINT64_MAX) {
+                if (instance->metadata.size != object->size)
+                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Size parsed from filename and manifest don't match, refusing.");
+        } else
+                instance->metadata.size = object->size;
+
+        if (instance->metadata.mode == MODE_INVALID)
+                instance->metadata.mode = object->mode;
+        if (instance->metadata.mtime == USEC_INFINITY && object->mtime_nsec < (NSEC_INFINITY / NSEC_PER_USEC) * NSEC_PER_USEC)
+                instance->metadata.mtime = object->mtime_nsec / NSEC_PER_USEC;
+        if (!instance->metadata.partition_uuid_set && object->gpt_uuid_set) {
+                instance->metadata.partition_uuid = object->gpt_uuid;
+                instance->metadata.partition_uuid_set = true;
+        }
+
+        if (!instance->metadata.partition_flags_set && object->gpt_flags_set) {
+                instance->metadata.partition_flags = object->gpt_flags;
+                instance->metadata.partition_flags_set = true;
+        }
+
+        if (instance->metadata.no_auto < 0 && object->gpt_no_auto >= 0)
+                instance->metadata.no_auto = object->gpt_no_auto;
+        if (instance->metadata.growfs < 0 && object->gpt_growfs >= 0)
+                instance->metadata.growfs = object->gpt_growfs;
+        if (instance->metadata.read_only < 0 && object->read_only >= 0)
+                instance->metadata.read_only = object->read_only;
+
+        return 0;
+}
+
+typedef struct Uapi16ProcessContext {
+        Resource *resource;
+        const GptPartitionType *match_partition_type;
+} Uapi16ProcessContext;
+
+static int uapi16_process_object(Uapi16Object *object, size_t index, void *userdata) {
+        Uapi16ProcessContext *pc = ASSERT_PTR(userdata);
+        Resource *rr = ASSERT_PTR(pc->resource);
+        int r;
+
+        assert(object);
+
+        /* Honour validity properties, but otherwise ignore intro object */
+        if (index == 0) {
+                r = uapi16_object_check_validity(object);
+                if (r < 0) {
+                        if (verify_freshness())
+                                return log_error_errno(r, "Manifest file marked invalid: %m");
+
+                        log_warning_errno(r, "Manifest file marked invalid, but freshness checks disabled, proceeding.");
+                }
+
+                return 0;
+        }
+
+        /* Ignore non-regular files */
+        if (!S_ISREG(object->type)) {
+                log_debug("Skipping file '%s' as it is not a regular file.", object->name);
+                return 0;
+        }
+
+        /* Ignore files whose partition type does not match the one of the partitions we'd write them to */
+        if (object->gpt_type_uuid_set && pc->match_partition_type &&
+            !sd_id128_equal(object->gpt_type_uuid, pc->match_partition_type->uuid)) {
+                log_debug("Skipping file '%s' as its partition type does not match the target's.", object->name);
+                return 0;
+        }
+
+        r = uapi16_object_check_validity(object);
+        if (r < 0) {
+                if (verify_freshness()) {
+                        log_warning_errno(r, "File '%s' marked invalid, skipping: %m", object->name);
+                        return 0;
+                }
+
+                log_warning_errno(r, "File '%s' marked invalid, but freshness checks disabled, proceeding.", object->name);
+        }
+
+        if (object->size == UINT64_MAX) {
+                log_warning_errno(SYNTHETIC_ERRNO(EBADMSG), "File '%s' lacks a size, skipping.", object->name);
+                return 0;
+        }
+
+        if (!iovec_is_set(&object->sha256)) {
+                log_warning_errno(SYNTHETIC_ERRNO(EBADMSG), "File '%s' lacks a SHA256 digest, skipping.", object->name);
+                return 0;
+        }
+
+        /* Refuse "%" for the same reason as in sha256sums_line() */
+        if (strchr(object->name, '%')) {
+                log_warning_errno(SYNTHETIC_ERRNO(EBADMSG), "File '%s' contains a '%%' character, skipping.", object->name);
+                return 0;
+        }
+
+        _cleanup_(instance_metadata_destroy) InstanceMetadata extracted_fields = INSTANCE_METADATA_NULL;
+        r = pattern_match_many(rr->patterns, object->name, &extracted_fields);
+        if (r < 0)
+                return log_error_errno(r, "Failed to match pattern: %m");
+        if (!IN_SET(r, PATTERN_MATCH_YES, PATTERN_MATCH_YES_AND_RETRY))
+                return 0;
+
+        _cleanup_free_ char *path = NULL;
+        r = import_url_append_component(rr->path, object->name, &path);
+        if (r < 0)
+                return log_error_errno(r, "Failed to build instance URL: %m");
+
+        Instance *instance;
+        r = resource_add_instance(rr, path, &extracted_fields, &instance);
+        if (r < 0)
+                return r;
+
+        /* Now copy the fields from the manifest into our instances logic */
+        return instance_copy_uapi16(instance, object);
+}
+
+static int uapi16_process(
+                Resource *rr,
+                const struct iovec *manifest,
+                const GptPartitionType *match_partition_type) {
+
+        assert(rr);
+        assert(manifest);
+
+        if (!iovec_is_set(manifest))
+                return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), "Manifest is empty, refusing.");
+
+        _cleanup_fclose_ FILE *f = fmemopen_unlocked(manifest->iov_base, manifest->iov_len, "r");
+        if (!f)
+                return log_error_errno(errno, "Failed to open manifest buffer as stream: %m");
+
+        return uapi16_manifest_read(
+                        f,
+                        /* certificates= */ NULL,
+                        /* n_certificates= */ 0,
+                        uapi16_process_object,
+                        &(Uapi16ProcessContext) {
+                                .resource = rr,
+                                .match_partition_type = match_partition_type,
+                        });
+}
+
 static int resource_load_from_web(
                 Resource *rr,
                 bool verify,
-                Hashmap **web_cache) {
+                Hashmap **web_cache,
+                const GptPartitionType *match_partition_type) {
 
         _cleanup_(iovec_done) struct iovec buf = {};
         struct iovec manifest = {};
@@ -639,7 +812,7 @@ static int resource_load_from_web(
         POINTER_MAY_BE_NULL(web_cache);
 
         /* Key the cache by the URL of the manifest itself, not the one of the resource, as the former
-         * depends on ManifestPath= too. */
+         * depends on ManifestFormat= and ManifestPath= too. */
         _cleanup_free_ char *manifest_url = NULL;
         r = resource_manifest_url(rr, &manifest_url);
         if (r < 0)
@@ -660,7 +833,19 @@ static int resource_load_from_web(
                 manifest = buf;
         }
 
-        r = sha256sums_process(rr, &manifest);
+        switch (rr->manifest_format) {
+
+        case MANIFEST_FORMAT_SHA256SUMS:
+                r = sha256sums_process(rr, &manifest);
+                break;
+
+        case MANIFEST_FORMAT_UAPI16:
+                r = uapi16_process(rr, &manifest, match_partition_type);
+                break;
+
+        default:
+                assert_not_reached();
+        }
         if (r < 0)
                 return r;
 
@@ -696,11 +881,19 @@ static int instance_cmp(Instance *const*a, Instance *const*b) {
         return path_compare((*a)->path, (*b)->path);
 }
 
-int resource_load_instances(Resource *rr, bool verify, Hashmap **web_cache) {
+int resource_load_instances(
+                Resource *rr,
+                bool verify,
+                Hashmap **web_cache,
+                const GptPartitionType *match_partition_type) {
         int r;
 
         assert(rr);
 
+        /* If match_partition_type is specified, files listed in UAPI.16 manifests that declare a different
+         * partition type are skipped. This is used to filter the sources by the partition type of the
+         * target the files would be written to. */
+
         switch (rr->type) {
 
         case RESOURCE_TAR:
@@ -719,7 +912,7 @@ int resource_load_instances(Resource *rr, bool verify, Hashmap **web_cache) {
 
         case RESOURCE_URL_FILE:
         case RESOURCE_URL_TAR:
-                r = resource_load_from_web(rr, verify, web_cache);
+                r = resource_load_from_web(rr, verify, web_cache, match_partition_type);
                 break;
 
         default:

src/sysupdate/sysupdate-resource.h

@@ -79,7 +79,7 @@ typedef struct Resource {
         char **patterns;
         GptPartitionType partition_type;
         bool partition_type_set;
-        ManifestFormat manifest_format;  /* Only MANIFEST_FORMAT_SHA256SUMS for now */
+        ManifestFormat manifest_format;  /* MANIFEST_FORMAT_SHA256SUMS or MANIFEST_FORMAT_UAPI16 */
         char *manifest_path;
 
         /* All instances of this resource we found */
@@ -94,7 +94,11 @@ void resource_destroy(Resource *rr);
 
 bool resource_has_glob_directory_pattern(Resource *rr);
 
-int resource_load_instances(Resource *rr, bool verify, Hashmap **web_cache);
+int resource_load_instances(
+                Resource *rr,
+                bool verify,
+                Hashmap **web_cache,
+                const GptPartitionType *match_partition_type);
 
 Instance* resource_find_instance(Resource *rr, const char *version);
 

src/sysupdate/sysupdate-transfer.c

@@ -638,7 +638,7 @@ static int transfer_finalize(Transfer *t, const char *origin, unsigned line, Has
                                                   "Source path is not a valid HTTP, HTTPS, file, or provider URL: %s", t->source.path);
         }
 
-        if (t->source.manifest_format != MANIFEST_FORMAT_SHA256SUMS)
+        if (!IN_SET(t->source.manifest_format, MANIFEST_FORMAT_SHA256SUMS, MANIFEST_FORMAT_UAPI16))
                 return log_syntax(NULL, LOG_ERR, origin, line, SYNTHETIC_ERRNO(EINVAL),
                                   "Manifest format '%s' not supported for source specifications, refusing.",
                                   manifest_format_to_string(t->source.manifest_format));

src/sysupdate/sysupdate.c

@@ -572,7 +572,8 @@ static int context_load_installed_instances(Context *c) {
                 r = resource_load_instances(
                                 &t->target,
                                 c->verify >= 0 ? c->verify : t->verify != 0,
-                                &c->web_cache);
+                                &c->web_cache,
+                                /* match_partition_type= */ NULL);
                 if (r < 0)
                         return r;
         }
@@ -583,7 +584,8 @@ static int context_load_installed_instances(Context *c) {
                 r = resource_load_instances(
                                 &t->target,
                                 c->verify >= 0 ? c->verify : t->verify != 0,
-                                &c->web_cache);
+                                &c->web_cache,
+                                /* match_partition_type= */ NULL);
                 if (r < 0)
                         return r;
         }
@@ -604,7 +606,8 @@ static int context_load_available_instances(Context *c) {
                 r = resource_load_instances(
                                 &t->source,
                                 c->verify >= 0 ? c->verify : t->verify != 0,
-                                &c->web_cache);
+                                &c->web_cache,
+                                t->target.partition_type_set ? &t->target.partition_type : NULL);
                 if (r < 0)
                         return r;
         }

test/units/TEST-72-SYSUPDATE.sh

@@ -1022,9 +1022,9 @@ cmp "$MP/payload/mp-v1.bin" "$MP/target/mp-v1.bin"
 # there is none
 sed 's|^ManifestPath=.*|&\nManifestPath=|' "$MP/definitions/01-mp.transfer" >"$MP/bad-definitions/01-mp.transfer"
 (! "$SYSUPDATE" --definitions="$MP/bad-definitions" --verify=no list)
-# Only SHA256SUMS manifests are supported for now
-sed 's|^ManifestFormat=.*|ManifestFormat=uapi16|' "$MP/definitions/01-mp.transfer" >"$MP/bad-definitions/01-mp.transfer"
-(! "$SYSUPDATE" --definitions="$MP/bad-definitions" --verify=no list) |& grep -F "Manifest format 'uapi16' not supported" >/dev/null
+# Only SHA256SUMS and UAPI.16 manifests are supported
+sed 's|^ManifestFormat=.*|ManifestFormat=mtree|' "$MP/definitions/01-mp.transfer" >"$MP/bad-definitions/01-mp.transfer"
+(! "$SYSUPDATE" --definitions="$MP/bad-definitions" --verify=no list) |& grep -F "Manifest format 'mtree' not supported" >/dev/null
 # ManifestPath= must be a URL, and is only supported for URL sources
 sed 's|^ManifestPath=.*|ManifestPath=/foo/SHA256SUMS|' "$MP/definitions/01-mp.transfer" >"$MP/bad-definitions/01-mp.transfer"
 (! "$SYSUPDATE" --definitions="$MP/bad-definitions" --verify=no list) |& grep -F "Manifest path is not a valid" >/dev/null