diff --git a/man/importctl.xml b/man/importctl.xml
index ae4e375359b..3083c2661af 100644
--- a/man/importctl.xml
+++ b/man/importctl.xml
@@ -114,17 +114,25 @@
URL, with its suffix removed.
The image is verified before it is made available, unless is
- specified. Verification is done either via a file with the name of the image and the
- suffix .sha256 and a detached .sha256.asc or
- .sha256.gpg signature or via separate SHA256SUMS and
- SHA256SUMS.asc or SHA256SUMS.gpg files. The signature
- files need to be made available on the same web server, under the same URL as the
- .tar file. With
+ specified. Verification is done via a UAPI.16 File Manifest file
+ Uapi16Manifest, which is tried first, or, if that does not exist, either via a
+ file with the name of the image and the suffix .sha256 and a detached
+ .sha256.asc or .sha256.gpg signature or via separate
+ SHA256SUMS and SHA256SUMS.asc or
+ SHA256SUMS.gpg files. These files need to be made available on the same web
+ server, under the same URL as the .tar file. With
, only the SHA256 checksum for the file is verified, based on the
- .sha256 suffixed file or the SHA256SUMS file. With
- , the sha checksum file is first verified with the detached GPG
- signature of .sha256 or SHA256SUMS. The public key for
- this verification step needs to be available in the system keyring consisting of
+ manifest, the .sha256 suffixed file or the SHA256SUMS
+ file. With , the checksum file is first verified with its
+ signature. UAPI.16 manifests carry an inline PKCS#7 signature, which is verified against the X.509
+ certificates found in the UAPI.11
+ Verification of OS Artifacts hierarchy, for the role manifest and the
+ context import (for example
+ /etc/voa/os/manifest/import/x509/*-certificate.pem).
+ For the other formats the detached GPG signature of .sha256 or
+ SHA256SUMS is verified. The public key for this verification step needs to be
+ available in the system keyring consisting of
/usr/lib/systemd/import-pubring.pgp and the local keyring in
/etc/systemd/import-pubring.pgp (or, if that does not exist,
/etc/systemd/import-pubring.gpg).
@@ -318,8 +326,10 @@
signature. If no, no verification is done. If
checksum is specified, the download is checked for integrity after the transfer is
complete, but no signatures are verified. If signature is specified, the checksum
- is verified and the image's signature is checked against a local keyring of trustable vendors, in
- combination with the vendor keyring in /usr/lib/systemd/import-pubring.pgp.
+ is verified and the image's signature is checked: the inline signature of a UAPI.16 manifest against
+ the certificates in the VOA hierarchy, and the detached GPG signature of other checksum files against
+ a local keyring of trustable vendors, in combination with the vendor keyring in
+ /usr/lib/systemd/import-pubring.pgp (see pull-tar above).
It is strongly recommended to set this option to signature if the server and
protocol support this. Defaults to signature.
diff --git a/src/basic/web-util.c b/src/basic/web-util.c
index 8d8c0d05ca0..0c74243cce4 100644
--- a/src/basic/web-util.c
+++ b/src/basic/web-util.c
@@ -1,6 +1,7 @@
/* SPDX-License-Identifier: LGPL-2.1-or-later */
#include "alloc-util.h"
+#include "parse-util.h"
#include "path-util.h"
#include "string-util.h"
#include "strv.h"
@@ -124,3 +125,73 @@ bool http_etag_is_valid(const char *etag) {
return true;
}
+
+int parse_content_range(const char *s, uint64_t *ret_start, uint64_t *ret_end, uint64_t *ret_total) {
+ uint64_t start, end, total;
+ int r;
+
+ assert(s);
+
+ /* Parses the value of a "Content-Range:" header of a 206 response, i.e. "bytes START-END/TOTAL"
+ * (RFC 9110, section 14.4). Only the "bytes" unit is supported. TOTAL may be "*" if the server
+ * doesn't know it, in which case UINT64_MAX is returned for it. The unsatisfied-range form of 416
+ * responses (where the range is replaced by an asterisk) is refused. All return parameters are
+ * optional. Note that the numbers are always decimal, even with leading zeros (which RFC 9110
+ * permits), hence explicitly parse them in base 10. */
+
+ const char *p = startswith_no_case(s, "bytes ");
+ if (!p)
+ return -EINVAL;
+
+ size_t n = strspn(p, DIGITS);
+ if (n == 0 || p[n] != '-')
+ return -EINVAL;
+
+ _cleanup_free_ char *a = strndup(p, n);
+ if (!a)
+ return -ENOMEM;
+
+ p += n + 1;
+ n = strspn(p, DIGITS);
+ if (n == 0 || p[n] != '/')
+ return -EINVAL;
+
+ _cleanup_free_ char *b = strndup(p, n);
+ if (!b)
+ return -ENOMEM;
+
+ p += n + 1;
+
+ r = safe_atou64_full(a, 10, &start);
+ if (r < 0)
+ return r;
+
+ r = safe_atou64_full(b, 10, &end);
+ if (r < 0)
+ return r;
+
+ if (end < start)
+ return -EINVAL;
+
+ if (streq(p, "*"))
+ total = UINT64_MAX;
+ else {
+ if (isempty(p) || !in_charset(p, DIGITS))
+ return -EINVAL;
+
+ r = safe_atou64_full(p, 10, &total);
+ if (r < 0)
+ return r;
+
+ if (end >= total)
+ return -EINVAL;
+ }
+
+ if (ret_start)
+ *ret_start = start;
+ if (ret_end)
+ *ret_end = end;
+ if (ret_total)
+ *ret_total = total;
+ return 0;
+}
diff --git a/src/basic/web-util.h b/src/basic/web-util.h
index d015383f46f..943b3b15bdf 100644
--- a/src/basic/web-util.h
+++ b/src/basic/web-util.h
@@ -12,3 +12,5 @@ int provider_url_parse(const char *url, char **ret_socket, char **ret_resource);
bool provider_url_is_valid(const char *url);
bool http_header_valid(const char *header) _pure_;
bool http_etag_is_valid(const char *etag) _pure_;
+
+int parse_content_range(const char *s, uint64_t *ret_start, uint64_t *ret_end, uint64_t *ret_total);
diff --git a/src/import/pull-common.c b/src/import/pull-common.c
index 500b6174a17..58e3db4ee11 100644
--- a/src/import/pull-common.c
+++ b/src/import/pull-common.c
@@ -25,9 +25,13 @@
#include "string-util.h"
#include "strv.h"
#include "tmpfile-util.h"
+#include "uapi16-manifest.h"
#include "web-util.h"
#define FILENAME_ESCAPE "/.#\"\'"
+
+/* The VOA context (see voa-util.h) the certificates to verify UAPI.16 manifests with are looked up in */
+#define IMPORT_VOA_CONTEXT "import"
#define HASH_URL_THRESHOLD_LENGTH (_POSIX_PATH_MAX - 16)
int pull_find_old_etags(
@@ -236,6 +240,74 @@ int pull_make_auxiliary_job(
return 0;
}
+static bool is_uapi16_manifest_file(const char *fn) {
+ /* Returns true if the specified filename refers to a UAPI.16 manifest */
+
+ if (!fn)
+ return false;
+
+ return streq(fn, UAPI16_MANIFEST_FILENAME) || uapi16_manifest_filename_is_suffixed(fn);
+}
+
+static bool pull_job_is_uapi16_manifest(PullJob *j) {
+ _cleanup_free_ char *fn = NULL;
+
+ /* Returns true if the specified job downloads a UAPI.16 manifest, whose signature is inline, and
+ * which hence needs no separate signature file */
+
+ if (!j || !j->url)
+ return false;
+
+ if (import_url_last_component(j->url, &fn) < 0)
+ return false;
+
+ return is_uapi16_manifest_file(fn);
+}
+
+bool pull_signature_job_deferred(PullJob *checksum_job, PullJob *signature_job) {
+ /* If the checksum job starts out with a UAPI.16 manifest, we only know whether we need the detached
+ * signature once it finished, since UAPI.16 manifests carry their signature inline. Otherwise the
+ * signature job can be started right away. */
+
+ return checksum_job && signature_job && pull_job_is_uapi16_manifest(checksum_job);
+}
+
+int pull_signature_job_begin(PullJob *checksum_job, PullJob **signature_job) {
+ int r;
+
+ assert(checksum_job);
+ assert(signature_job);
+
+ /* Only do something if the signature job was deferred, see pull_signature_job_deferred() */
+ if (!*signature_job || (*signature_job)->state != PULL_JOB_INIT)
+ return 0;
+
+ /* UAPI.16 manifests carry their signature inline, hence drop the signature job without ever starting
+ * it in that case */
+ if (pull_job_is_uapi16_manifest(checksum_job)) {
+ *signature_job = pull_job_unref(*signature_job);
+ return 0;
+ }
+
+ /* If we fell back to SHA256SUMS, go directly for its signature, instead of trying the per-file
+ * signature files first, which cannot match. */
+ VerificationStyle style;
+ r = verification_style_from_url(checksum_job->url, &style);
+ if (r < 0)
+ return r;
+ if (style == VERIFICATION_PER_DIRECTORY) {
+ _cleanup_free_ char *url = NULL;
+
+ r = import_url_change_last_component(checksum_job->url, "SHA256SUMS.gpg", &url);
+ if (r < 0)
+ return r;
+
+ free_and_replace((*signature_job)->url, url);
+ }
+
+ return pull_job_begin(*signature_job);
+}
+
static bool is_checksum_file(const char *fn, ImportFlags flags) {
/* Returns true if the specified filename refers to a checksum file we grok. If the caller told us
* explicitly whether it is one, we follow that, otherwise we decide based on the name. */
@@ -250,7 +322,7 @@ static bool is_checksum_file(const char *fn, ImportFlags flags) {
if (!fn)
return false;
- return streq(fn, "SHA256SUMS") || endswith(fn, ".sha256");
+ return streq(fn, "SHA256SUMS") || endswith(fn, ".sha256") || is_uapi16_manifest_file(fn);
}
static SignatureStyle signature_style_from_filename(const char *fn) {
@@ -351,16 +423,43 @@ int pull_make_verification_jobs(
* to acquire isn't a checksum or signature file anyway */
if (!checksum_file && signature_style_from_filename(fn) < 0) {
_cleanup_free_ char *checksum_url = NULL;
- const char *suffixed = NULL;
- /* Queue jobs for the checksum file for the image. */
+ /* Queue jobs for the checksum file for the image. Start with a UAPI.16 manifest, then try
+ * the suse-style checksum (if there's a base filename), then the ubuntu-style one, see
+ * pull_job_restart_with_sha256sum(). If signature verification is requested, UAPI.16
+ * manifests are only useful if certificates to verify them with are installed. If there
+ * are none, skip them, and go directly for the SHA256SUMS flavours, which are verified with
+ * gpg. */
- if (fn)
- suffixed = strjoina(fn, ".sha256"); /* Start with the suse-style checksum (if there's a base filename) */
+ X509 **certificates = NULL;
+ size_t n_certificates = 0;
+ CLEANUP_ARRAY(certificates, n_certificates, uapi16_certificates_free);
+
+ /* If the main download itself is named like a UAPI.16 manifest (but was declared not to be a
+ * checksum file), a manifest next to it would be the very same file, hence skip it. */
+ bool try_uapi16 = !is_uapi16_manifest_file(fn);
+ if (try_uapi16 && verify == IMPORT_VERIFY_SIGNATURE) {
+ /* Failing to load the certificates is not fatal: we can still fall back to the
+ * gpg-signed checksum files. (The failure has been logged already by the loader.) */
+ r = uapi16_load_voa_certificates(XAT_FDROOT, IMPORT_VOA_CONTEXT, &certificates, &n_certificates);
+ if (r < 0) {
+ log_debug_errno(r, "Failed to load certificates for verifying UAPI.16 manifests, not trying to acquire one: %m");
+ try_uapi16 = false;
+ } else if (n_certificates == 0) {
+ log_debug("No certificates for verifying UAPI.16 manifests installed, not trying to acquire one.");
+ try_uapi16 = false;
+ }
+ }
+
+ const char *checksum_fn;
+ if (try_uapi16)
+ checksum_fn = UAPI16_MANIFEST_FILENAME;
+ else if (fn)
+ checksum_fn = strjoina(fn, ".sha256");
else
- suffixed = "SHA256SUMS";
+ checksum_fn = "SHA256SUMS";
- r = import_url_change_last_component(url, suffixed, &checksum_url);
+ r = import_url_change_last_component(url, checksum_fn, &checksum_url);
if (r < 0)
return r;
@@ -368,12 +467,27 @@ int pull_make_verification_jobs(
if (r < 0)
return r;
+ /* Keep the certificates around for verifying the manifest, so that we load them only once */
+ checksum_job->uapi16_certificates = TAKE_PTR(certificates);
+ checksum_job->n_uapi16_certificates = TAKE_GENERIC(n_certificates, size_t, 0);
+
+ r = strdup_to(&checksum_job->main_filename, fn);
+ if (r < 0)
+ return r;
+
checksum_job->on_finished = on_finished;
checksum_job->uncompressed_max = checksum_job->compressed_max = 1ULL * 1024ULL * 1024ULL;
- checksum_job->on_not_found = pull_job_restart_with_sha256sum; /* if this fails, look for ubuntu-style checksum */
+ checksum_job->on_not_found = pull_job_restart_with_sha256sum;
}
- if (verify == IMPORT_VERIFY_SIGNATURE && signature_style_from_filename(fn) < 0) {
+ /* Acquire the detached signature file if signature verification is requested. If the checksum job
+ * starts out with a UAPI.16 manifest, this job is only started once that one finished, and only if
+ * it fell back to a different checksum file, since UAPI.16 manifests carry their signature inline,
+ * see pull_signature_job_begin(). UAPI.16 manifests that are the main download (and are treated as
+ * checksum files) need no detached signature file either. */
+ if (verify == IMPORT_VERIFY_SIGNATURE &&
+ signature_style_from_filename(fn) < 0 &&
+ !(checksum_file && is_uapi16_manifest_file(fn))) {
_cleanup_free_ char *signature_url = NULL;
PullJobNotFound on_not_found;
const char *suffixed;
@@ -479,6 +593,170 @@ static int verify_one(PullJob *checksum_job, PullJob *job, ImportFlags flags) {
return 1;
}
+typedef struct Uapi16VerifyItem {
+ PullJob *job;
+ char *filename;
+ bool verified;
+} Uapi16VerifyItem;
+
+typedef struct Uapi16VerifyContext {
+ Uapi16VerifyItem *items;
+ size_t n_items;
+} Uapi16VerifyContext;
+
+static void uapi16_verify_context_done(Uapi16VerifyContext *c) {
+ assert(c);
+
+ FOREACH_ARRAY(i, c->items, c->n_items)
+ free(i->filename);
+
+ c->items = mfree(c->items);
+ c->n_items = 0;
+}
+
+static int uapi16_verify_object(Uapi16Object *o, size_t index, void *userdata) {
+ Uapi16VerifyContext *c = ASSERT_PTR(userdata);
+ int r;
+
+ assert(o);
+
+ /* The leading object describes the manifest itself: honour its validity properties */
+ if (index == 0) {
+ r = uapi16_object_check_validity(o);
+ if (r < 0)
+ return log_error_errno(r, "DOWNLOAD INVALID: Manifest is marked invalid: %m");
+
+ return 0;
+ }
+
+ FOREACH_ARRAY(i, c->items, c->n_items) {
+ if (!streq_ptr(o->name, i->filename))
+ continue;
+
+ r = uapi16_object_check_validity(o);
+ if (r < 0)
+ return log_error_errno(r, "DOWNLOAD INVALID: %s is marked invalid in manifest: %m", i->filename);
+
+ if (!iovec_is_set(&o->sha256))
+ return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
+ "DOWNLOAD INVALID: Manifest entry for %s carries no SHA256 digest.", i->filename);
+
+ if (iovec_memcmp(&o->sha256, &i->job->checksum) != 0)
+ return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
+ "DOWNLOAD INVALID: Checksum of %s file did not check out, file has been tampered with.", i->filename);
+
+ i->verified = true;
+ }
+
+ return 0;
+}
+
+static int verify_uapi16(
+ ImportVerify verify,
+ ImportFlags flags,
+ PullJob *manifest_job,
+ PullJob *const *jobs,
+ size_t n_jobs) {
+
+ _cleanup_(uapi16_verify_context_done) Uapi16VerifyContext c = {};
+ int r;
+
+ assert(IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE));
+ assert(manifest_job);
+ assert(jobs || n_jobs == 0);
+
+ /* Verifies the specified jobs against the UAPI.16 manifest downloaded by manifest_job, and the
+ * manifest itself against its inline PKCS#7 signature, if signature verification is requested. */
+
+ c.items = new0(Uapi16VerifyItem, n_jobs);
+ if (!c.items)
+ return log_oom();
+
+ FOREACH_ARRAY(j, jobs, n_jobs) {
+ _cleanup_free_ char *fn = NULL;
+
+ if (!*j)
+ continue;
+
+ assert(IN_SET((*j)->state, PULL_JOB_DONE, PULL_JOB_FAILED));
+
+ /* Don't verify the checksum if we didn't actually successfully download something new */
+ if ((*j)->state != PULL_JOB_DONE || (*j)->error != 0 || (*j)->etag_exists)
+ continue;
+
+ assert((*j)->calc_checksum);
+ assert(iovec_is_set(&(*j)->checksum));
+
+ r = import_url_last_component((*j)->url, &fn);
+ if (r < 0)
+ return log_error_errno(r, "Failed to extract filename from URL '%s': %m", (*j)->url);
+
+ if (!filename_is_valid(fn))
+ return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
+ "Cannot verify checksum, could not determine server-side file name.");
+
+ /* Whether the main download (the first job) is a checksum file might have been specified
+ * explicitly, but the auxiliary files are always classified by their name. */
+ if (is_checksum_file(fn, j == jobs ? flags : 0) || signature_style_from_filename(fn) >= 0)
+ return log_error_errno(SYNTHETIC_ERRNO(ELOOP),
+ "Cannot verify checksum/signature files via themselves.");
+
+ c.items[c.n_items++] = (Uapi16VerifyItem) {
+ .job = *j,
+ .filename = TAKE_PTR(fn),
+ };
+ }
+
+ /* If the manifest was acquired as checksum file, the certificates have been loaded already when the
+ * job was created. If it is the main download, load them now. */
+ if (verify == IMPORT_VERIFY_SIGNATURE) {
+ if (manifest_job->n_uapi16_certificates == 0) {
+ r = uapi16_load_voa_certificates(
+ XAT_FDROOT,
+ IMPORT_VOA_CONTEXT,
+ &manifest_job->uapi16_certificates,
+ &manifest_job->n_uapi16_certificates);
+ if (r < 0)
+ return r; /* already logged */
+ }
+ if (manifest_job->n_uapi16_certificates == 0)
+ return log_error_errno(SYNTHETIC_ERRNO(ENOKEY),
+ "Signature verification of UAPI.16 manifest requested, but no certificates found in the VOA hierarchy (role '%s', context '%s'), refusing. (Try --verify=checksum?)",
+ UAPI16_VOA_ROLE, IMPORT_VOA_CONTEXT);
+ }
+
+ if (!iovec_is_set(&manifest_job->payload))
+ return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), "Manifest is empty, cannot verify.");
+
+ _cleanup_fclose_ FILE *f = fmemopen_unlocked(manifest_job->payload.iov_base, manifest_job->payload.iov_len, "r");
+ if (!f)
+ return log_error_errno(errno, "Failed to open manifest as stream: %m");
+
+ /* Note that the signature is only verified at the very end, hence only trust the results once this
+ * returned successfully. */
+ r = uapi16_manifest_read(
+ f,
+ verify == IMPORT_VERIFY_SIGNATURE ? manifest_job->uapi16_certificates : NULL,
+ verify == IMPORT_VERIFY_SIGNATURE ? manifest_job->n_uapi16_certificates : 0,
+ uapi16_verify_object,
+ &c);
+ if (r < 0)
+ return r; /* already logged */
+
+ if (verify == IMPORT_VERIFY_SIGNATURE)
+ log_info("UAPI.16 manifest signature verification succeeded.");
+
+ FOREACH_ARRAY(i, c.items, c.n_items) {
+ if (!i->verified)
+ return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
+ "DOWNLOAD INVALID: %s is not listed in UAPI.16 manifest, cannot verify.", i->filename);
+
+ log_info("SHA256 checksum of %s is valid.", i->job->url);
+ }
+
+ return 0;
+}
+
static int verify_gpg(
const struct iovec *payload,
const struct iovec *signature) {
@@ -708,6 +986,19 @@ int pull_verify(ImportVerify verify,
assert(checksum_job);
assert(checksum_job->state == PULL_JOB_DONE);
+ /* UAPI.16 manifests carry their signature inline, hence verify them in one go */
+ if (pull_job_is_uapi16_manifest(checksum_job)) {
+ PullJob *const jobs[] = {
+ main_job,
+ settings_job,
+ roothash_job,
+ roothash_signature_job,
+ verity_job,
+ };
+
+ return verify_uapi16(verify, flags, checksum_job, jobs, ELEMENTSOF(jobs));
+ }
+
if (!iovec_is_set(&checksum_job->payload))
return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
"Checksum is empty, cannot verify.");
@@ -732,6 +1023,11 @@ int pull_verify(ImportVerify verify,
return 0;
assert(verify_job);
+
+ /* If the main download is a UAPI.16 manifest, verify its inline signature */
+ if (pull_job_is_uapi16_manifest(verify_job))
+ return verify_uapi16(verify, flags, verify_job, /* jobs= */ NULL, /* n_jobs= */ 0);
+
assert(signature_job);
assert(signature_job->state == PULL_JOB_DONE);
@@ -755,6 +1051,11 @@ int verification_style_from_url(const char *url, VerificationStyle *ret) {
if (r < 0)
return r;
+ if (streq(last, UAPI16_MANIFEST_FILENAME)) {
+ *ret = VERIFICATION_UAPI16_MANIFEST;
+ return 0;
+ }
+
if (streq(last, "SHA256SUMS")) {
*ret = VERIFICATION_PER_DIRECTORY;
return 0;
@@ -775,7 +1076,9 @@ int pull_job_restart_with_sha256sum(PullJob *j, char **ret) {
assert(j);
assert(ret);
- /* Generic implementation of a PullJobNotFound handler, that restarts the job requesting SHA256SUMS */
+ /* Generic implementation of a PullJobNotFound handler, that restarts the job requesting the next
+ * checksum file: after the UAPI.16 manifest the *.sha256 file (if we know the file name of the
+ * download), and then SHA256SUMS. */
r = verification_style_from_url(j->url, &style);
if (r < 0)
@@ -786,6 +1089,20 @@ int pull_job_restart_with_sha256sum(PullJob *j, char **ret) {
return 0;
}
+ if (style == VERIFICATION_UAPI16_MANIFEST) {
+ _cleanup_free_ char *next = j->main_filename ? strjoin(j->main_filename, ".sha256") : strdup("SHA256SUMS");
+ if (!next)
+ return log_oom();
+
+ log_debug("Got 404 for '%s', now trying to get %s instead.", j->url, next);
+
+ r = import_url_change_last_component(j->url, next, ret);
+ if (r < 0)
+ return log_error_errno(r, "Failed to replace %s suffix: %m", UAPI16_MANIFEST_FILENAME);
+
+ return 1;
+ }
+
assert(style == VERIFICATION_PER_FILE); /* This must have been .sha256 style URL before */
log_debug("Got 404 for '%s', now trying to get SHA256SUMS instead.", j->url);
diff --git a/src/import/pull-common.h b/src/import/pull-common.h
index 710ba2280aa..bedf39e810e 100644
--- a/src/import/pull-common.h
+++ b/src/import/pull-common.h
@@ -55,8 +55,9 @@ int pull_verify(
PullJob *verity_job);
typedef enum VerificationStyle {
- VERIFICATION_PER_FILE, /* SUSE-style ".sha256" files with detached gpg signature */
- VERIFICATION_PER_DIRECTORY, /* Ubuntu-style SHA256SUM files with detached SHA256SUM.gpg signatures */
+ VERIFICATION_UAPI16_MANIFEST, /* UAPI.16 "Uapi16Manifest" files with inline PKCS#7 signatures */
+ VERIFICATION_PER_FILE, /* SUSE-style ".sha256" files with detached gpg signature */
+ VERIFICATION_PER_DIRECTORY, /* Ubuntu-style SHA256SUM files with detached SHA256SUM.gpg signatures */
_VERIFICATION_STYLE_MAX,
_VERIFICATION_STYLE_INVALID = -EINVAL,
} VerificationStyle;
@@ -74,6 +75,14 @@ typedef enum SignatureStyle {
int signature_style_from_url(const char *url, SignatureStyle *ret, char **ret_filename);
+/* Returns true if the signature job shall not be started right away, but only once the checksum job
+ * finished, via pull_signature_job_begin() */
+bool pull_signature_job_deferred(PullJob *checksum_job, PullJob *signature_job);
+
+/* To be called once the checksum job finished successfully: starts the signature job if it was deferred,
+ * unless the checksum file turned out to be a UAPI.16 manifest, in which case the signature job is dropped */
+int pull_signature_job_begin(PullJob *checksum_job, PullJob **signature_job);
+
int pull_job_restart_with_sha256sum(PullJob *job, char **ret);
int pull_job_restart_with_signature(PullJob *job, char **ret);
diff --git a/src/import/pull-job.c b/src/import/pull-job.c
index 17a3d829021..507d105f9f8 100644
--- a/src/import/pull-job.c
+++ b/src/import/pull-job.c
@@ -25,6 +25,7 @@
#include "strv.h"
#include "sync-util.h"
#include "time-util.h"
+#include "uapi16-manifest.h"
#include "web-util.h"
#include "xattr-util.h"
@@ -78,6 +79,8 @@ PullJob* pull_job_unref(PullJob *j) {
sym_EVP_MD_CTX_free(j->checksum_ctx);
free(j->url);
+ free(j->main_filename);
+ uapi16_certificates_free(j->uapi16_certificates, j->n_uapi16_certificates);
free(j->etag);
strv_free(j->old_etags);
iovec_done(&j->payload);
@@ -107,8 +110,11 @@ static int pull_job_finish(PullJob *j, int ret) {
if (IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED))
return 0;
- /* Stop listening for further data, if we are using the Varlink transport */
- j->provider_source = sd_event_source_disable_unref(j->provider_source);
+ /* Stop listening for further data and release the connection, if we are using the Varlink
+ * transport. This matters if we finish early (i.e. once we acquired the requested source range),
+ * since otherwise the provider would remain blocked writing to us until the job is freed. The curl
+ * transport similarly aborts the transfer in that case. */
+ pull_job_provider_done(j);
if (ret == 0) {
j->state = PULL_JOB_DONE;
@@ -144,6 +150,8 @@ int pull_job_restart(PullJob *j, const char *new_url) {
iovec_done(&j->payload);
j->written_compressed = 0;
j->written_uncompressed = 0;
+ j->decoded_total = 0;
+ j->range_requested = j->range_applied = false;
j->content_length = UINT64_MAX;
j->etag = mfree(j->etag);
j->etag_exists = false;
@@ -191,6 +199,29 @@ static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata)
assert(p);
assert(sz > 0);
+ /* Only pass on the requested range of the decoded data, i.e. skip over everything before it and
+ * drop everything after it. Note that the dropped data is still covered by the checksum of the
+ * download, as that is calculated from the encoded data. */
+ if (j->decoded_total > UINT64_MAX - sz)
+ return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow");
+
+ uint64_t pos = j->decoded_total;
+ j->decoded_total += sz;
+
+ if (j->decoded_total <= j->source_offset)
+ return 0;
+ if (pos < j->source_offset) {
+ p = (const uint8_t*) p + (j->source_offset - pos);
+ sz -= j->source_offset - pos;
+ }
+
+ if (j->source_size != UINT64_MAX) {
+ if (j->written_uncompressed >= j->source_size)
+ return 0;
+
+ sz = MIN(sz, j->source_size - j->written_uncompressed);
+ }
+
if (j->written_uncompressed > UINT64_MAX - sz)
return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow");
@@ -347,7 +378,19 @@ static int pull_job_begin_running(PullJob *j) {
return 0;
}
-static int pull_job_complete(PullJob *j) {
+static bool pull_job_source_range_done(PullJob *j) {
+ assert(j);
+
+ /* Returns true if we have written out the complete requested source range, and the rest of the
+ * download is of no interest to us, since we need no checksum of it as a whole */
+
+ return j->state == PULL_JOB_RUNNING &&
+ !j->checksum_ctx &&
+ j->source_size != UINT64_MAX &&
+ j->written_uncompressed >= j->source_size;
+}
+
+static int pull_job_complete(PullJob *j, bool early) {
int r;
assert(j);
@@ -370,15 +413,23 @@ static int pull_job_complete(PullJob *j) {
if (j->state != PULL_JOB_RUNNING)
return pull_job_finish(j, log_error_errno(SYNTHETIC_ERRNO(EIO), "Premature connection termination."));
- /* Finalize decompressor. */
- r = decompressor_push(j->compress, /* data= */ NULL, /* size= */ 0, pull_job_write_uncompressed, j);
- if (r < 0)
- return pull_job_finish(j, r);
+ /* If we stop early, after the requested source range, the stream is incomplete by design, hence
+ * neither finalize the decompressor nor check the length in that case. */
+ if (!early) {
+ /* Finalize decompressor. */
+ r = decompressor_push(j->compress, /* data= */ NULL, /* size= */ 0, pull_job_write_uncompressed, j);
+ if (r < 0)
+ return pull_job_finish(j, r);
- uint64_t cl = pull_job_content_length_effective(j);
- if (cl != UINT64_MAX &&
- cl != j->written_compressed)
- return pull_job_finish(j, log_error_errno(SYNTHETIC_ERRNO(EIO), "Download truncated."));
+ if (j->decoded_total < j->source_offset ||
+ (j->source_size != UINT64_MAX && j->written_uncompressed < j->source_size))
+ return pull_job_finish(j, log_error_errno(SYNTHETIC_ERRNO(EBADMSG), "Decoded data of %s ends before the requested range does, refusing.", pull_job_description(j)));
+
+ uint64_t cl = pull_job_content_length_effective(j);
+ if (cl != UINT64_MAX &&
+ cl != j->written_compressed)
+ return pull_job_finish(j, log_error_errno(SYNTHETIC_ERRNO(EIO), "Download truncated."));
+ }
if (j->checksum_ctx) {
unsigned checksum_len;
@@ -553,7 +604,7 @@ static int pull_job_curl_on_finished(CurlSlot *slot, CURL *curl, CURLcode result
return pull_job_finish(j, log_error_errno(SYNTHETIC_ERRNO(EIO), "HTTP request to %s finished with unexpected code %li.", j->url, status));
}
- return pull_job_complete(j);
+ return pull_job_complete(j, /* early= */ false);
}
static bool pull_job_http_status_ok(PullJob *j) {
@@ -608,6 +659,8 @@ static int pull_job_detect_compression(PullJob *j) {
}
static int pull_job_process_data(PullJob *j, const void *p, size_t sz) {
+ int r;
+
assert(j);
assert(p || sz == 0);
@@ -623,7 +676,20 @@ static int pull_job_process_data(PullJob *j, const void *p, size_t sz) {
return pull_job_detect_compression(j);
case PULL_JOB_RUNNING:
- return pull_job_write_compressed(j, &IOVEC_MAKE((void*) p, sz));
+ r = pull_job_write_compressed(j, &IOVEC_MAKE((void*) p, sz));
+ if (r < 0)
+ return r;
+
+ /* Once we have the requested source range, there's no need to download (and decode) the
+ * rest, unless we need the checksum of the whole download. Don't do this for HTTP error
+ * responses though, whose body is not what we asked for: such a transfer has to run to
+ * its end, so that it fails with the HTTP error. */
+ if (pull_job_http_status_ok(j) && pull_job_source_range_done(j)) {
+ log_debug("Acquired requested range of %s, ending transfer.", pull_job_description(j));
+ return pull_job_complete(j, /* early= */ true);
+ }
+
+ return 0;
case PULL_JOB_DONE:
case PULL_JOB_FAILED:
@@ -641,15 +707,49 @@ static size_t pull_job_write_callback(void *contents, size_t size, size_t nmemb,
assert(contents);
+ /* If the server replied with partial content we must know where it starts. (If it replied with the
+ * full content instead, we cut out the range ourselves.) */
+ if (j->range_requested && !j->range_applied) {
+ long status;
+
+ if (sym_curl_easy_getinfo(curl_slot_get_easy(j->slot), CURLINFO_RESPONSE_CODE, &status) == CURLE_OK &&
+ status == 206) {
+ pull_job_finish(j, log_error_errno(SYNTHETIC_ERRNO(EPROTO), "Server returned partial content without range information for %s, refusing.", pull_job_description(j)));
+ return 0;
+ }
+ }
+
r = pull_job_process_data(j, contents, sz);
if (r < 0) {
pull_job_finish(j, r);
return 0;
}
+ /* If we are done already (because we have everything we need), abort the transfer */
+ if (PULL_JOB_IS_COMPLETE(j))
+ return 0;
+
return sz;
}
+static bool pull_job_content_range_ok(PullJob *j, uint64_t start, uint64_t end, uint64_t total) {
+ assert(j);
+
+ /* Checks whether the range the server returned is the one we asked for */
+
+ if (!j->range_requested || start != j->source_offset)
+ return false;
+
+ /* For an open-ended request the server must return everything up to the end of the source,
+ * which we can only tell if it tells us the total size */
+ if (j->source_size == UINT64_MAX)
+ return total != UINT64_MAX && end == total - 1;
+
+ /* For a closed range it must not return more than we asked for. (If it returns less, because the
+ * source is shorter, we'll notice once the download finished.) */
+ return end <= j->source_offset + j->source_size - 1;
+}
+
static size_t pull_job_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
_cleanup_free_ char *length = NULL, *last_modified = NULL, *etag = NULL, *ct = NULL;
size_t sz = size * nmemb;
@@ -686,6 +786,35 @@ static size_t pull_job_header_callback(void *contents, size_t size, size_t nmemb
return sz;
}
+ if (status == 206) {
+ _cleanup_free_ char *range = NULL;
+
+ r = curl_header_strdup(contents, sz, "Content-Range:", &range);
+ if (r < 0) {
+ log_oom();
+ goto fail;
+ }
+ if (r > 0) {
+ /* The server sent us only the range we asked for, hence the data we receive starts
+ * at its beginning rather than at the start of the source. */
+ uint64_t start, end, total;
+ r = parse_content_range(range, &start, &end, &total);
+ if (r == -ENOMEM) {
+ log_oom();
+ goto fail;
+ }
+ if (r < 0 || !pull_job_content_range_ok(j, start, end, total)) {
+ r = log_error_errno(SYNTHETIC_ERRNO(EPROTO), "Server returned unexpected range '%s' for %s, refusing.", range, pull_job_description(j));
+ goto fail;
+ }
+
+ log_debug("Server honoured range request for %s, starting at offset %" PRIu64 ".", pull_job_description(j), start);
+ j->decoded_total = start;
+ j->range_applied = true;
+ return sz;
+ }
+ }
+
if (http_status_ok(status) || http_status_etag_exists(status)) {
/* Check Etag on OK and etag exists responses. */
@@ -860,6 +989,7 @@ int pull_job_new(
.uncompressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */
.url = TAKE_PTR(u),
.offset = UINT64_MAX,
+ .source_size = UINT64_MAX,
.sync = true,
.expected_content_length = UINT64_MAX,
};
@@ -906,11 +1036,13 @@ static int pull_job_provider_on_io(sd_event_source *s, int fd, uint32_t revents,
return pull_job_finish(j, log_error_errno(errno, "Failed to read from resource provider for %s: %m", j->url));
}
if (n == 0) /* EOF: the provider has sent everything */
- return pull_job_complete(j);
+ return pull_job_complete(j, /* early= */ false);
r = pull_job_process_data(j, buffer, (size_t) n);
if (r < 0)
return pull_job_finish(j, r);
+ if (PULL_JOB_IS_COMPLETE(j)) /* We have everything we need already */
+ return 0;
pull_job_report_progress(j, pull_job_content_length_effective(j), j->written_compressed);
return 0;
@@ -1102,6 +1234,31 @@ int pull_job_begin(PullJob *j) {
return -EIO;
}
+ /* If we only need a range of the data, let's ask the server for just that. This is only possible if
+ * the data is not encoded (as we cannot decode from the middle of a stream), and if we don't have to
+ * verify the download as a whole. Only do this for HTTP, where we can tell from the response whether
+ * the server complied, and fall back to cutting out the range ourselves if it did not. */
+ if (j->allow_range_request &&
+ j->compression == COMPRESSION_NONE &&
+ (j->source_offset > 0 || j->source_size != UINT64_MAX) &&
+ j->source_size != 0 &&
+ http_url_is_valid(j->url)) {
+ _cleanup_free_ char *range = NULL;
+
+ if (j->source_size == UINT64_MAX)
+ r = asprintf(&range, "%" PRIu64 "-", j->source_offset);
+ else
+ r = asprintf(&range, "%" PRIu64 "-%" PRIu64, j->source_offset, j->source_offset + j->source_size - 1);
+ if (r < 0)
+ return -ENOMEM;
+
+ if (sym_curl_easy_setopt(easy, CURLOPT_RANGE, range) != CURLE_OK)
+ return -EIO;
+
+ log_debug("Requesting range %s of %s.", range, pull_job_description(j));
+ j->range_requested = true;
+ }
+
if (sym_curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, pull_job_write_callback) != CURLE_OK)
return -EIO;
diff --git a/src/import/pull-job.h b/src/import/pull-job.h
index 10ca9185c97..98fcd0abb1b 100644
--- a/src/import/pull-job.h
+++ b/src/import/pull-job.h
@@ -56,6 +56,18 @@ typedef struct PullJob {
uint64_t written_uncompressed;
uint64_t offset;
+ /* The range of the decoded data to write out (anything before and after it is dropped), and the
+ * number of decoded bytes seen so far, including the dropped ones */
+ uint64_t source_offset;
+ uint64_t source_size;
+ uint64_t decoded_total;
+
+ /* Whether we may ask an HTTP server for just the source range (i.e. if the data is not encoded and
+ * the download needs not be verified as a whole), whether we did, and whether the server complied */
+ bool allow_range_request;
+ bool range_requested;
+ bool range_applied;
+
uint64_t uncompressed_max;
uint64_t compressed_max;
@@ -80,6 +92,14 @@ typedef struct PullJob {
bool calc_checksum;
EVP_MD_CTX *checksum_ctx;
+ /* For checksum jobs: the file name of the download to verify, to derive the names of alternative
+ * checksum files to try from */
+ char *main_filename;
+
+ /* For UAPI.16 manifest jobs: the certificates to verify the inline signature with */
+ X509 **uapi16_certificates;
+ size_t n_uapi16_certificates;
+
struct iovec checksum;
struct iovec expected_checksum;
diff --git a/src/import/pull-raw.c b/src/import/pull-raw.c
index bc6b8c00181..45134aa7a3f 100644
--- a/src/import/pull-raw.c
+++ b/src/import/pull-raw.c
@@ -532,6 +532,15 @@ static void raw_pull_job_on_finished(PullJob *j) {
*
* We only do something when we got all files */
+ /* If the signature job was deferred, only now we know whether we need the detached signature file */
+ if (j == p->checksum_job) {
+ r = pull_signature_job_begin(p->checksum_job, &p->signature_job);
+ if (r < 0) {
+ r = log_error_errno(r, "Failed to start signature job: %m");
+ goto finish;
+ }
+ }
+
if (!raw_pull_is_done(p))
return;
@@ -792,6 +801,8 @@ int raw_pull_start(
const char *local,
uint64_t offset,
uint64_t size_max,
+ uint64_t source_offset,
+ uint64_t source_size,
Compression compression,
ImportFlags flags,
ImportVerify verify,
@@ -809,6 +820,8 @@ int raw_pull_start(
assert(!(flags & ~IMPORT_PULL_FLAGS_MASK_RAW));
assert(offset == UINT64_MAX || FLAGS_SET(flags, IMPORT_DIRECT));
assert(!FLAGS_SET(flags, IMPORT_ZERO_FILL) || (offset != UINT64_MAX && size_max != UINT64_MAX));
+ assert((source_offset == 0 && source_size == UINT64_MAX) || FLAGS_SET(flags, IMPORT_DIRECT));
+ assert((source_offset == 0 && source_size == UINT64_MAX) || !IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE));
assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !(flags & IMPORT_DIRECT));
assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !iovec_is_set(checksum));
@@ -869,6 +882,10 @@ int raw_pull_start(
if (offset != UINT64_MAX)
p->raw_job->offset = p->offset = offset;
p->raw_job->zero_fill = FLAGS_SET(flags, IMPORT_ZERO_FILL);
+ p->raw_job->source_offset = source_offset;
+ p->raw_job->source_size = source_size;
+ /* Only a download that is not verified as a whole may be acquired partially */
+ p->raw_job->allow_range_request = verify == IMPORT_VERIFY_NO && !iovec_is_set(checksum);
/* A previously downloaded image is neither keyed by the compression algorithm nor was it
* necessarily verified against the decoded digest, hence don't reuse it if either is requested. */
@@ -971,6 +988,10 @@ int raw_pull_start(
j->on_progress = raw_pull_job_on_progress;
j->sync = FLAGS_SET(flags, IMPORT_SYNC);
+ /* The signature job might only be started once the checksum job finished, see raw_pull_job_on_finished() */
+ if (j == p->signature_job && pull_signature_job_deferred(p->checksum_job, p->signature_job))
+ continue;
+
r = pull_job_begin(j);
if (r < 0)
return r;
diff --git a/src/import/pull-raw.h b/src/import/pull-raw.h
index 466a1bd4b43..b3ca1ae6d2e 100644
--- a/src/import/pull-raw.h
+++ b/src/import/pull-raw.h
@@ -11,4 +11,4 @@ RawPull* raw_pull_unref(RawPull *p);
DEFINE_TRIVIAL_CLEANUP_FUNC(RawPull*, raw_pull_unref);
-int raw_pull_start(RawPull *p, const char *url, const char *local, uint64_t offset, uint64_t size_max, Compression compression, ImportFlags flags, ImportVerify verify, const struct iovec *checksum, const struct iovec *decoded_checksum);
+int raw_pull_start(RawPull *p, const char *url, const char *local, uint64_t offset, uint64_t size_max, uint64_t source_offset, uint64_t source_size, Compression compression, ImportFlags flags, ImportVerify verify, const struct iovec *checksum, const struct iovec *decoded_checksum);
diff --git a/src/import/pull-tar.c b/src/import/pull-tar.c
index e716cb90418..7f92d3119a3 100644
--- a/src/import/pull-tar.c
+++ b/src/import/pull-tar.c
@@ -436,6 +436,15 @@ static void tar_pull_job_on_finished(PullJob *j) {
/* This is invoked if either the download completed successfully, or the download was skipped because
* we already have the etag. */
+ /* If the signature job was deferred, only now we know whether we need the detached signature file */
+ if (j == p->checksum_job) {
+ r = pull_signature_job_begin(p->checksum_job, &p->signature_job);
+ if (r < 0) {
+ r = log_error_errno(r, "Failed to start signature job: %m");
+ goto finish;
+ }
+ }
+
if (!tar_pull_is_done(p))
return;
@@ -805,6 +814,10 @@ int tar_pull_start(
j->on_progress = tar_pull_job_on_progress;
j->sync = FLAGS_SET(flags, IMPORT_SYNC);
+ /* The signature job might only be started once the checksum job finished, see tar_pull_job_on_finished() */
+ if (j == p->signature_job && pull_signature_job_deferred(p->checksum_job, p->signature_job))
+ continue;
+
r = pull_job_begin(j);
if (r < 0)
return r;
diff --git a/src/import/pull.c b/src/import/pull.c
index b6b64b23e69..b4acc852396 100644
--- a/src/import/pull.c
+++ b/src/import/pull.c
@@ -35,6 +35,7 @@ static char *arg_image_root = NULL;
static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE;
static ImportFlags arg_import_flags = IMPORT_PULL_SETTINGS | IMPORT_PULL_ROOTHASH | IMPORT_PULL_ROOTHASH_SIGNATURE | IMPORT_PULL_VERITY | IMPORT_BTRFS_SUBVOL | IMPORT_BTRFS_QUOTA | IMPORT_CONVERT_QCOW2 | IMPORT_SYNC;
static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX;
+static uint64_t arg_source_offset = 0, arg_source_size = UINT64_MAX;
static Compression arg_compression = _COMPRESSION_INVALID;
static struct iovec arg_checksum = {};
static struct iovec arg_decoded_checksum = {};
@@ -117,6 +118,20 @@ static int normalize_local(const char *local, const char *url, char **ret) {
return 0;
}
+static int refuse_ranges(const char *type) {
+ assert(type);
+
+ /* Ranges only make sense for raw images, hence refuse them for anything else */
+
+ if (arg_offset != UINT64_MAX || arg_size_max != UINT64_MAX)
+ return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), "--offset=/--size-max= are not supported for %s images.", type);
+
+ if (arg_source_offset != 0 || arg_source_size != UINT64_MAX)
+ return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), "--source-offset=/--source-size= are not supported for %s images.", type);
+
+ return 0;
+}
+
static void on_tar_finished(TarPull *pull, int error, void *userdata) {
sd_event *event = userdata;
assert(pull);
@@ -161,6 +176,10 @@ static int verb_tar(int argc, char *argv[], uintptr_t _data, void *userdata) {
if (arg_import_flags & (IMPORT_PULL_CHECKSUM_FILE_YES|IMPORT_PULL_CHECKSUM_FILE_NO))
return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), "--checksum-file= is not supported for tar images.");
+ r = refuse_ranges("tar");
+ if (r < 0)
+ return r;
+
r = normalize_local(local, url, &normalized);
if (r < 0)
return r;
@@ -249,6 +268,8 @@ static int verb_raw(int argc, char *argv[], uintptr_t _data, void *userdata) {
normalized,
arg_offset,
arg_size_max,
+ arg_source_offset,
+ arg_source_size,
arg_compression,
arg_import_flags & IMPORT_PULL_FLAGS_MASK_RAW,
arg_verify,
@@ -286,6 +307,10 @@ static int verb_oci(int argc, char *argv[], uintptr_t _data, void *userdata) {
if (arg_import_flags & (IMPORT_PULL_CHECKSUM_FILE_YES|IMPORT_PULL_CHECKSUM_FILE_NO))
return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), "--checksum-file= is not supported for OCI images.");
+ r = refuse_ranges("OCI");
+ if (r < 0)
+ return r;
+
_cleanup_free_ char *image = NULL;
r = oci_ref_parse(ref, /* ret_registry= */ NULL, &image, /* ret_tag= */ NULL);
if (r == -EINVAL)
@@ -398,7 +423,7 @@ static int parse_argv(int argc, char *argv[], char ***ret_args) {
}
OPTION_LONG("verify-decoded", "SHA256",
- "Verify the decoded (i.e. decompressed) image against the specified SHA256 hash"):
+ "Verify the decoded (i.e. decompressed, and cut to the source range) image against the specified SHA256 hash"):
r = import_parse_sha256_argument("--verify-decoded=", opts.arg, &arg_decoded_checksum);
if (r < 0)
return r;
@@ -529,6 +554,32 @@ static int parse_argv(int argc, char *argv[], char ***ret_args) {
break;
}
+ OPTION_LONG("source-offset", "BYTES", "Offset of the range of the decoded data to write"): {
+ uint64_t u;
+
+ r = safe_atou64(opts.arg, &u);
+ if (r < 0)
+ return log_error_errno(r, "Failed to parse --source-offset= argument: %s", opts.arg);
+ if (!FILE_SIZE_VALID(u))
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --source-offset= switch too large: %s", opts.arg);
+
+ arg_source_offset = u;
+ break;
+ }
+
+ OPTION_LONG("source-size", "BYTES", "Size of the range of the decoded data to write"): {
+ uint64_t u;
+
+ r = safe_atou64(opts.arg, &u);
+ if (r < 0)
+ return log_error_errno(r, "Failed to parse --source-size= argument: %s", opts.arg);
+ if (!FILE_SIZE_VALID(u))
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --source-size= switch too large: %s", opts.arg);
+
+ arg_source_size = u;
+ break;
+ }
+
OPTION_LONG("compression", "ALGORITHM",
"Compression algorithm of the downloaded data, one of: 'auto', 'none', 'gzip', 'xz', 'zstd', 'bzip2', 'lz4'"):
r = import_parse_compression_argument("--compression=", opts.arg, &arg_compression);
@@ -574,6 +625,19 @@ static int parse_argv(int argc, char *argv[], char ***ret_args) {
if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_import_flags, IMPORT_DIRECT))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode.");
+ if ((arg_source_offset != 0 || arg_source_size != UINT64_MAX) && !FLAGS_SET(arg_import_flags, IMPORT_DIRECT))
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Source range only supported in --direct mode.");
+
+ if (arg_source_size != UINT64_MAX && arg_source_size > UINT64_MAX - arg_source_offset)
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Source offset and size out of range.");
+
+ /* A manifest (UAPI.16 or SHA256SUMS) authenticates the image as a whole, which we cannot vouch for
+ * if only a part of it is written. */
+ if ((arg_source_offset != 0 || arg_source_size != UINT64_MAX) &&
+ IN_SET(arg_verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE))
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+ "Source range not supported with manifest based verification, use --verify=no or --verify= (together with --verify-decoded=).");
+
if (FLAGS_SET(arg_import_flags, IMPORT_ZERO_FILL) && (arg_offset == UINT64_MAX || arg_size_max == UINT64_MAX))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Zero filling only supported if both --offset= and --size-max= are specified.");
diff --git a/src/test/test-web-util.c b/src/test/test-web-util.c
index c64f1e0046f..77491e9ce3e 100644
--- a/src/test/test-web-util.c
+++ b/src/test/test-web-util.c
@@ -54,4 +54,73 @@ TEST(provider_url_parse) {
test_provider_url_parse_one("provider:/run/foobar/waldo", /* socket= */ NULL, /* resource= */ NULL);
}
+TEST(parse_content_range) {
+ uint64_t start, end, total;
+
+ ASSERT_OK(parse_content_range("bytes 0-9/10", &start, &end, &total));
+ ASSERT_EQ(start, 0U);
+ ASSERT_EQ(end, 9U);
+ ASSERT_EQ(total, 10U);
+
+ ASSERT_OK(parse_content_range("bytes 1000-5999/262164", &start, &end, &total));
+ ASSERT_EQ(start, 1000U);
+ ASSERT_EQ(end, 5999U);
+ ASSERT_EQ(total, 262164U);
+
+ /* Single byte ranges, case insensitive unit, unknown total */
+ ASSERT_OK(parse_content_range("Bytes 7-7/*", &start, &end, &total));
+ ASSERT_EQ(start, 7U);
+ ASSERT_EQ(end, 7U);
+ ASSERT_EQ(total, UINT64_MAX);
+
+ ASSERT_OK(parse_content_range("bytes 0-18446744073709551613/18446744073709551614", &start, &end, &total));
+ ASSERT_EQ(start, 0U);
+ ASSERT_EQ(end, UINT64_MAX - 2);
+ ASSERT_EQ(total, UINT64_MAX - 1);
+
+ /* Leading zeros are permitted, and the numbers are always decimal, never octal */
+ ASSERT_OK(parse_content_range("bytes 0100-0177/01000", &start, &end, &total));
+ ASSERT_EQ(start, 100U);
+ ASSERT_EQ(end, 177U);
+ ASSERT_EQ(total, 1000U);
+
+ ASSERT_OK(parse_content_range("bytes 08-09/010", &start, &end, &total));
+ ASSERT_EQ(start, 8U);
+ ASSERT_EQ(end, 9U);
+ ASSERT_EQ(total, 10U);
+
+ /* All return parameters are optional */
+ ASSERT_OK(parse_content_range("bytes 0-9/10", /* ret_start= */ NULL, /* ret_end= */ NULL, /* ret_total= */ NULL));
+ start = end = 0;
+ ASSERT_OK(parse_content_range("bytes 3-4/*", &start, &end, /* ret_total= */ NULL));
+ ASSERT_EQ(start, 3U);
+ ASSERT_EQ(end, 4U);
+ ASSERT_ERROR(parse_content_range("bytes 5-4/10", /* ret_start= */ NULL, /* ret_end= */ NULL, /* ret_total= */ NULL), EINVAL);
+
+ /* End before start, or beyond the total */
+ ASSERT_ERROR(parse_content_range("bytes 5-4/10", &start, &end, &total), EINVAL);
+ ASSERT_ERROR(parse_content_range("bytes 0-10/10", &start, &end, &total), EINVAL);
+ ASSERT_ERROR(parse_content_range("bytes 0-0/0", &start, &end, &total), EINVAL);
+
+ /* The unsatisfied range form of 416 responses, other units, and malformed values */
+ ASSERT_ERROR(parse_content_range("bytes */10", &start, &end, &total), EINVAL);
+ ASSERT_ERROR(parse_content_range("items 0-1/2", &start, &end, &total), EINVAL);
+ ASSERT_ERROR(parse_content_range("bytes", &start, &end, &total), EINVAL);
+ ASSERT_ERROR(parse_content_range("bytes ", &start, &end, &total), EINVAL);
+ ASSERT_ERROR(parse_content_range("bytes 0-1", &start, &end, &total), EINVAL);
+ ASSERT_ERROR(parse_content_range("bytes 0-1/", &start, &end, &total), EINVAL);
+ ASSERT_ERROR(parse_content_range("bytes -1/2", &start, &end, &total), EINVAL);
+ ASSERT_ERROR(parse_content_range("bytes 0-/2", &start, &end, &total), EINVAL);
+ ASSERT_ERROR(parse_content_range("bytes 0-1/2x", &start, &end, &total), EINVAL);
+ ASSERT_ERROR(parse_content_range("bytes 0-1/**", &start, &end, &total), EINVAL);
+ ASSERT_ERROR(parse_content_range("bytes 0-1/2", &start, &end, &total), EINVAL);
+ ASSERT_ERROR(parse_content_range("bytes +0-1/2", &start, &end, &total), EINVAL);
+ ASSERT_ERROR(parse_content_range("bytes 0-1/-2", &start, &end, &total), EINVAL);
+ ASSERT_ERROR(parse_content_range("bytes 0-1/2 ", &start, &end, &total), EINVAL);
+
+ /* Overflow */
+ ASSERT_ERROR(parse_content_range("bytes 0-18446744073709551616/*", &start, &end, &total), ERANGE);
+ ASSERT_ERROR(parse_content_range("bytes 0-1/18446744073709551616", &start, &end, &total), ERANGE);
+}
+
DEFINE_TEST_MAIN(LOG_INFO);
diff --git a/test/units/TEST-25-IMPORT.sh b/test/units/TEST-25-IMPORT.sh
index c42978b9a15..952761864ad 100755
--- a/test/units/TEST-25-IMPORT.sh
+++ b/test/units/TEST-25-IMPORT.sh
@@ -299,6 +299,314 @@ test ! -e "$COMPDIR/root/bad"
(! "$PULL" tar --image-root="$COMPDIR/root" --keep-download=no --settings=no --verify=no --verify-decoded="$WRONG_HASH" "file://$COMPDIR/tree.tar.gz" bad)
test ! -e "$COMPDIR/root/bad"
+# → systemd-pull can extract a range of the decoded data ← #
+
+PAYLOAD_SIZE="$(stat -c %s "$COMPDIR/payload.raw")"
+dd if="$COMPDIR/payload.raw" of="$COMPDIR/slice.raw" iflag=skip_bytes,count_bytes skip=1000 count=5000 status=none
+dd if="$COMPDIR/payload.raw" of="$COMPDIR/tail.raw" iflag=skip_bytes skip=1000 status=none
+dd if="$COMPDIR/payload.raw" of="$COMPDIR/head.raw" iflag=count_bytes count=5000 status=none
+dd if="$COMPDIR/payload.raw" of="$COMPDIR/end.raw" iflag=skip_bytes skip=$((PAYLOAD_SIZE - 10)) status=none
+SLICE_HASH="$(sha256sum "$COMPDIR/slice.raw" | cut -d' ' -f1)"
+
+# --verify-decoded= covers just the range
+"$PULL" raw --verify=no --direct --source-offset=1000 --source-size=5000 --verify-decoded="$SLICE_HASH" "file://$COMPDIR/payload.raw" "$COMPDIR/out.raw"
+cmp "$COMPDIR/slice.raw" "$COMPDIR/out.raw"
+rm "$COMPDIR/out.raw"
+(! "$PULL" raw --verify=no --direct --source-offset=1000 --source-size=5000 --verify-decoded="$PAYLOAD_HASH" "file://$COMPDIR/payload.raw" "$COMPDIR/out.raw")
+rm -f "$COMPDIR/out.raw"
+
+# Without a size the range extends to the end, without an offset it starts at the beginning
+"$PULL" raw --verify=no --direct --source-offset=1000 "file://$COMPDIR/payload.raw" "$COMPDIR/out.raw"
+cmp "$COMPDIR/tail.raw" "$COMPDIR/out.raw"
+rm "$COMPDIR/out.raw"
+"$PULL" raw --verify=no --direct --source-size=5000 "file://$COMPDIR/payload.raw" "$COMPDIR/out.raw"
+cmp "$COMPDIR/head.raw" "$COMPDIR/out.raw"
+rm "$COMPDIR/out.raw"
+
+# A range ending right at the end of the data is fine, one extending beyond it is refused
+"$PULL" raw --verify=no --direct --source-offset=$((PAYLOAD_SIZE - 10)) --source-size=10 "file://$COMPDIR/payload.raw" "$COMPDIR/out.raw"
+cmp "$COMPDIR/end.raw" "$COMPDIR/out.raw"
+rm "$COMPDIR/out.raw"
+(! "$PULL" raw --verify=no --direct --source-offset=$((PAYLOAD_SIZE - 10)) --source-size=11 "file://$COMPDIR/payload.raw" "$COMPDIR/out.raw")
+rm -f "$COMPDIR/out.raw"
+(! "$PULL" raw --verify=no --direct --source-offset=$((PAYLOAD_SIZE + 1)) "file://$COMPDIR/payload.raw" "$COMPDIR/out.raw")
+rm -f "$COMPDIR/out.raw"
+
+# The range refers to the decoded data, hence works the same for compressed downloads
+if command -v gzip >/dev/null; then
+ gzip -c "$COMPDIR/payload.raw" >"$COMPDIR/payload-range.raw.gz"
+ "$PULL" raw --verify=no --direct --compression=gzip --source-offset=1000 --source-size=5000 --verify-decoded="$SLICE_HASH" "file://$COMPDIR/payload-range.raw.gz" "$COMPDIR/out.raw"
+ cmp "$COMPDIR/slice.raw" "$COMPDIR/out.raw"
+ rm "$COMPDIR/out.raw"
+fi
+
+# Once the range is acquired the rest of the download is skipped, unless it is needed for verification
+SYSTEMD_LOG_LEVEL=debug "$PULL" raw --verify=no --direct --source-size=5000 "file://$COMPDIR/payload.raw" "$COMPDIR/out.raw" 2>"$COMPDIR/pull.log"
+cmp "$COMPDIR/head.raw" "$COMPDIR/out.raw"
+rm "$COMPDIR/out.raw"
+grep -F "Acquired requested range" "$COMPDIR/pull.log" >/dev/null
+SYSTEMD_LOG_LEVEL=debug "$PULL" raw --verify="$PAYLOAD_HASH" --direct --source-size=5000 "file://$COMPDIR/payload.raw" "$COMPDIR/out.raw" 2>"$COMPDIR/pull.log"
+cmp "$COMPDIR/head.raw" "$COMPDIR/out.raw"
+rm "$COMPDIR/out.raw"
+(! grep -F "Acquired requested range" "$COMPDIR/pull.log")
+
+# Ranges make no sense for tarballs and OCI images, and are refused
+(! "$PULL" tar --verify=no --direct --source-offset=1000 "file://$COMPDIR/tree.tar" "$COMPDIR/tardir") |& \
+ grep -F "not supported for tar images" >/dev/null
+(! "$PULL" tar --verify=no --direct --source-size=1000 "file://$COMPDIR/tree.tar" "$COMPDIR/tardir") |& \
+ grep -F "not supported for tar images" >/dev/null
+test ! -e "$COMPDIR/tardir"
+(! "$PULL" oci --verify=no --direct --source-offset=1000 foo/bar "$COMPDIR/ocidir") |& grep -F "not supported for OCI images" >/dev/null
+(! "$PULL" oci --verify=no --direct --source-size=1000 foo/bar "$COMPDIR/ocidir") |& grep -F "not supported for OCI images" >/dev/null
+
+# Ranges are only supported in --direct mode, and must not overflow
+(! "$PULL" raw --image-root="$COMPDIR/root" --verify=no --source-offset=1000 "file://$COMPDIR/payload.raw" bad)
+test ! -e "$COMPDIR/root/bad.raw"
+(! "$PULL" raw --verify=no --direct --source-offset=18446744073709551615 --source-size=2 "file://$COMPDIR/payload.raw" "$COMPDIR/out.raw")
+test ! -e "$COMPDIR/out.raw"
+
+# A manifest cannot authenticate a partial image, hence ranges are refused with manifest based verification
+for v in checksum signature; do
+ (! "$PULL" raw --verify="$v" --direct --source-offset=1000 "file://$COMPDIR/payload.raw" "$COMPDIR/out.raw") |& \
+ grep -F "Source range not supported with manifest based verification" >/dev/null
+ (! "$PULL" raw --verify="$v" --direct --source-size=5000 "file://$COMPDIR/payload.raw" "$COMPDIR/out.raw") |& \
+ grep -F "Source range not supported with manifest based verification" >/dev/null
+ test ! -e "$COMPDIR/out.raw"
+done
+
+# → For unencoded data systemd-pull asks HTTP servers for just the range, and cuts it out itself if they
+# do not honour the request ← #
+
+# A minimal HTTP server, where the first path component selects how range requests are handled: "honour"
+# returns the requested range, "ignore" always the whole file, "bogus" a range at the wrong offset, "short"
+# a range one byte shorter than requested, "long" one byte longer than requested, and "unknown" the
+# requested range, but without the total size. Each request is logged with its "Range:" header.
+RANGE_LOG="$COMPDIR/range.log"
+cat >"$COMPDIR/range-server.py" <<'EOF'
+import http.server, os, re, sys
+
+port, root, log = int(sys.argv[1]), sys.argv[2], sys.argv[3]
+
+class Handler(http.server.BaseHTTPRequestHandler):
+ def do_GET(self):
+ mode, _, name = self.path.lstrip('/').partition('/')
+ rng = self.headers.get('Range')
+ with open(log, 'a') as f:
+ print(mode, name, rng or '-', file=f)
+
+ try:
+ with open(os.path.join(root, os.path.basename(name)), 'rb') as f:
+ data = f.read()
+ except FileNotFoundError:
+ self.send_error(404)
+ return
+
+ m = re.fullmatch(r'bytes=(\d+)-(\d*)', rng or '')
+ if mode == 'ignore' or not m:
+ self.send_response(200)
+ chunk = data
+ else:
+ start = int(m.group(1))
+ end = int(m.group(2)) if m.group(2) else len(data) - 1
+ if mode == 'short':
+ end -= 1
+ elif mode == 'long':
+ end += 1
+ chunk = data[start:end + 1]
+ if mode == 'bogus':
+ start += 1
+ total = '*' if mode == 'unknown' else len(data)
+ self.send_response(206)
+ self.send_header('Content-Range', f'bytes {start}-{start + len(chunk) - 1}/{total}')
+ self.send_header('Content-Length', str(len(chunk)))
+ self.end_headers()
+ self.wfile.write(chunk)
+
+http.server.HTTPServer(('127.0.0.1', port), Handler).serve_forever()
+EOF
+systemd-run --unit=test-25-range-server -p Type=exec python3 "$COMPDIR/range-server.py" 8025 "$COMPDIR" "$RANGE_LOG"
+timeout 30 bash -c 'until curl -sf -o /dev/null http://127.0.0.1:8025/ignore/head.raw; do sleep .5; done'
+URL=http://127.0.0.1:8025
+
+for mode in honour ignore; do
+ : >"$RANGE_LOG"
+ "$PULL" raw --verify=no --compression=none --direct --source-offset=1000 --source-size=5000 --verify-decoded="$SLICE_HASH" "$URL/$mode/payload.raw" "$COMPDIR/out.raw"
+ cmp "$COMPDIR/slice.raw" "$COMPDIR/out.raw"
+ rm "$COMPDIR/out.raw"
+ grep -x "$mode payload.raw bytes=1000-5999" "$RANGE_LOG" >/dev/null
+
+ : >"$RANGE_LOG"
+ "$PULL" raw --verify=no --compression=none --direct --source-offset=1000 "$URL/$mode/payload.raw" "$COMPDIR/out.raw"
+ cmp "$COMPDIR/tail.raw" "$COMPDIR/out.raw"
+ rm "$COMPDIR/out.raw"
+ grep -x "$mode payload.raw bytes=1000-" "$RANGE_LOG" >/dev/null
+done
+
+# A range starting elsewhere than requested is refused
+(! "$PULL" raw --verify=no --compression=none --direct --source-offset=1000 --source-size=5000 "$URL/bogus/payload.raw" "$COMPDIR/out.raw") |& \
+ grep -F "Server returned unexpected range" >/dev/null
+rm -f "$COMPDIR/out.raw"
+
+# So is a range extending beyond the requested one
+(! "$PULL" raw --verify=no --compression=none --direct --source-offset=1000 --source-size=5000 "$URL/long/payload.raw" "$COMPDIR/out.raw") |& \
+ grep -F "Server returned unexpected range" >/dev/null
+rm -f "$COMPDIR/out.raw"
+
+# For an open-ended request the range must extend to the end of the source, which requires the total size
+for mode in short unknown; do
+ (! "$PULL" raw --verify=no --compression=none --direct --source-offset=1000 "$URL/$mode/payload.raw" "$COMPDIR/out.raw") |& \
+ grep -F "Server returned unexpected range" >/dev/null
+ rm -f "$COMPDIR/out.raw"
+done
+
+# For a closed range an unknown total size is fine
+"$PULL" raw --verify=no --compression=none --direct --source-offset=1000 --source-size=5000 "$URL/unknown/payload.raw" "$COMPDIR/out.raw"
+cmp "$COMPDIR/slice.raw" "$COMPDIR/out.raw"
+rm "$COMPDIR/out.raw"
+
+# No range is requested if the data might be encoded, or if the download is verified as a whole
+: >"$RANGE_LOG"
+"$PULL" raw --verify=no --direct --source-offset=1000 --source-size=5000 "$URL/honour/payload.raw" "$COMPDIR/out.raw"
+cmp "$COMPDIR/slice.raw" "$COMPDIR/out.raw"
+rm "$COMPDIR/out.raw"
+"$PULL" raw --verify="$PAYLOAD_HASH" --compression=none --direct --source-offset=1000 --source-size=5000 "$URL/honour/payload.raw" "$COMPDIR/out.raw"
+cmp "$COMPDIR/slice.raw" "$COMPDIR/out.raw"
+rm "$COMPDIR/out.raw"
+(! grep -v -x "honour payload.raw -" "$RANGE_LOG")
+
+systemctl stop test-25-range-server
+
+# → systemd-pull verifies downloads against UAPI.16 manifests, which are tried before *.sha256/SHA256SUMS ← #
+
+SRV="$COMPDIR/srv"
+mkdir -p "$SRV" "$COMPDIR/other"
+cp "$COMPDIR/payload.raw" "$SRV/payload.raw"
+systemd-manifest make "$SRV" >"$COMPDIR/Uapi16Manifest"
+mv "$COMPDIR/Uapi16Manifest" "$SRV/Uapi16Manifest"
+# A wrong SHA256SUMS file next to it is not consulted, as the manifest takes precedence
+echo "$WRONG_HASH payload.raw" >"$SRV/SHA256SUMS"
+"$PULL" raw --verify=checksum --direct "file://$SRV/payload.raw" "$COMPDIR/out.raw"
+cmp "$COMPDIR/payload.raw" "$COMPDIR/out.raw"
+rm "$COMPDIR/out.raw"
+
+# A manifest describing different contents is refused
+echo "something else" >"$COMPDIR/other/payload.raw"
+systemd-manifest make "$COMPDIR/other" >"$SRV/Uapi16Manifest"
+(! "$PULL" raw --verify=checksum --direct "file://$SRV/payload.raw" "$COMPDIR/out.raw")
+rm -f "$COMPDIR/out.raw"
+
+# Without a manifest SHA256SUMS is used
+rm "$SRV/Uapi16Manifest"
+(! "$PULL" raw --verify=checksum --direct "file://$SRV/payload.raw" "$COMPDIR/out.raw")
+rm -f "$COMPDIR/out.raw"
+(cd "$SRV" && sha256sum payload.raw >SHA256SUMS)
+"$PULL" raw --verify=checksum --direct "file://$SRV/payload.raw" "$COMPDIR/out.raw"
+rm "$COMPDIR/out.raw"
+
+# The same for tarballs
+TSRV="$COMPDIR/tsrv"
+mkdir -p "$TSRV" "$COMPDIR/tother"
+cp "$COMPDIR/tree.tar" "$TSRV/tree.tar"
+systemd-manifest make "$TSRV" >"$COMPDIR/Uapi16Manifest"
+mv "$COMPDIR/Uapi16Manifest" "$TSRV/Uapi16Manifest"
+"$PULL" tar --verify=checksum --direct "file://$TSRV/tree.tar" "$COMPDIR/tardir"
+diff -r "$COMPDIR/tree" "$COMPDIR/tardir"
+rm -rf "$COMPDIR/tardir"
+echo "something else" >"$COMPDIR/tother/tree.tar"
+systemd-manifest make "$COMPDIR/tother" >"$TSRV/Uapi16Manifest"
+(! "$PULL" tar --verify=checksum --direct "file://$TSRV/tree.tar" "$COMPDIR/tardir")
+rm -rf "$COMPDIR/tardir"
+
+# gpg signatures of SHA256SUMS are used if there are no certificates for UAPI.16 manifests (in which case
+# manifests are not tried at all), or if there are, but there's no manifest. (The checksum in the latter case
+# also implies the signature job is only started once the checksum file is acquired.)
+HAVE_GPG=0
+if command -v gpg >/dev/null; then
+ # verify_gpg() relies on the import semantics of gpg 2.4
+ GPG_VERSION="$(gpg --version | sed -n '1p' | awk '{print $NF}')"
+ GPG_REST="${GPG_VERSION#*.}"
+ if [[ "${GPG_VERSION%%.*}" -gt 2 ]] || { [[ "${GPG_VERSION%%.*}" -eq 2 ]] && [[ "${GPG_REST%%.*}" -ge 4 ]]; }; then
+ HAVE_GPG=1
+ fi
+fi
+if [[ "$HAVE_GPG" -eq 1 ]]; then
+ GPGHOME="$COMPDIR/gpghome"
+ mkdir -m 0700 "$GPGHOME"
+ GNUPGHOME="$GPGHOME" gpg --batch --pinentry-mode loopback --passphrase '' \
+ --quick-gen-key 'Test Key ' rsa2048 cert,sign never
+ GNUPGHOME="$GPGHOME" gpg --export --output "$COMPDIR/keyring.gpg"
+ GNUPGHOME="$GPGHOME" gpg --batch --pinentry-mode loopback --passphrase '' \
+ --detach-sign --yes --output "$SRV/SHA256SUMS.gpg" "$SRV/SHA256SUMS"
+
+ SYSTEMD_OPENPGP_KEYRING="$COMPDIR/keyring.gpg" "$PULL" raw --verify=signature --direct "file://$SRV/payload.raw" "$COMPDIR/out.raw"
+ cmp "$COMPDIR/payload.raw" "$COMPDIR/out.raw"
+ rm "$COMPDIR/out.raw"
+
+ # A bad signature is refused
+ echo "garbage" >>"$SRV/SHA256SUMS.gpg"
+ (! SYSTEMD_OPENPGP_KEYRING="$COMPDIR/keyring.gpg" "$PULL" raw --verify=signature --direct "file://$SRV/payload.raw" "$COMPDIR/out.raw")
+ rm -f "$COMPDIR/out.raw"
+ GNUPGHOME="$GPGHOME" gpg --batch --pinentry-mode loopback --passphrase '' \
+ --detach-sign --yes --output "$SRV/SHA256SUMS.gpg" "$SRV/SHA256SUMS"
+fi
+
+# The inline signature of the manifest is verified against the certificates in the VOA hierarchy (role
+# "manifest", context "import"), no detached gpg signature is needed
+if command -v openssl >/dev/null; then
+ VOA_OS="$(. /etc/os-release; echo "${ID:-linux}")"
+ VOA_DIR="/run/voa/$VOA_OS/manifest/import/x509"
+ openssl req -x509 -newkey rsa:2048 -nodes -keyout "$COMPDIR/good.key" -out "$COMPDIR/good.crt" -subj /CN=good -days 2 2>/dev/null
+ openssl req -x509 -newkey rsa:2048 -nodes -keyout "$COMPDIR/other.key" -out "$COMPDIR/other.crt" -subj /CN=other -days 2 2>/dev/null
+ # Certificates are installed, but there's no manifest: SHA256SUMS is used, and its signature is
+ # acquired directly, without trying the per-file signature names first
+ mkdir -p "$VOA_DIR"
+ cp "$COMPDIR/good.crt" "$VOA_DIR/test-25-certificate.pem"
+ if [[ "$HAVE_GPG" -eq 1 ]]; then
+ SYSTEMD_LOG_LEVEL=debug SYSTEMD_OPENPGP_KEYRING="$COMPDIR/keyring.gpg" \
+ "$PULL" raw --verify=signature --direct "file://$SRV/payload.raw" "$COMPDIR/out.raw" 2>"$COMPDIR/pull.log"
+ cmp "$COMPDIR/payload.raw" "$COMPDIR/out.raw"
+ rm "$COMPDIR/out.raw"
+ grep -F "Got 404 for 'file://$SRV/Uapi16Manifest'" "$COMPDIR/pull.log" >/dev/null
+ (! grep -F "payload.raw.sha256.asc" "$COMPDIR/pull.log")
+ rm "$SRV/SHA256SUMS.gpg"
+ fi
+ rm "$VOA_DIR/test-25-certificate.pem"
+
+ rm "$SRV/SHA256SUMS"
+ systemd-manifest make --private-key="$COMPDIR/good.key" --certificate="$COMPDIR/good.crt" "$SRV" >"$COMPDIR/Uapi16Manifest"
+ mv "$COMPDIR/Uapi16Manifest" "$SRV/Uapi16Manifest"
+
+ # No certificates installed: the manifest is not tried at all, and there's no other checksum file
+ (! "$PULL" raw --verify=signature --direct "file://$SRV/payload.raw" "$COMPDIR/out.raw") |& \
+ grep -F "Failed to retrieve SHA256 checksum" >/dev/null
+ rm -f "$COMPDIR/out.raw"
+
+ # The wrong certificate
+ cp "$COMPDIR/other.crt" "$VOA_DIR/test-25-certificate.pem"
+ (! "$PULL" raw --verify=signature --direct "file://$SRV/payload.raw" "$COMPDIR/out.raw")
+ rm -f "$COMPDIR/out.raw"
+
+ # The right certificate
+ cp "$COMPDIR/good.crt" "$VOA_DIR/test-25-certificate.pem"
+ "$PULL" raw --verify=signature --direct "file://$SRV/payload.raw" "$COMPDIR/out.raw"
+ cmp "$COMPDIR/payload.raw" "$COMPDIR/out.raw"
+ rm "$COMPDIR/out.raw"
+
+ # A manifest that is the main download is verified via its own inline signature
+ "$PULL" raw --verify=signature --direct "file://$SRV/Uapi16Manifest" "$COMPDIR/out.manifest"
+ rm "$COMPDIR/out.manifest"
+
+ # An unsigned manifest is refused if signatures are verified, but fine for checksum verification
+ systemd-manifest make "$SRV" >"$COMPDIR/Uapi16Manifest"
+ mv "$COMPDIR/Uapi16Manifest" "$SRV/Uapi16Manifest"
+ (! "$PULL" raw --verify=signature --direct "file://$SRV/payload.raw" "$COMPDIR/out.raw")
+ rm -f "$COMPDIR/out.raw"
+ "$PULL" raw --verify=checksum --direct "file://$SRV/payload.raw" "$COMPDIR/out.raw"
+ rm "$COMPDIR/out.raw"
+
+ rm -r "/run/voa/$VOA_OS/manifest/import"
+fi
+
rm -rf "$COMPDIR"
# → Now, let's test --checksum-file=, i.e. whether the downloaded file is treated as checksum file ← #
@@ -326,6 +634,11 @@ pull_cf() {
(! "$PULL" tar --verify=no --checksum-file=no "file://$CFDIR/foo.tar" foo) |& grep -F "not supported for tar images" >/dev/null
(! "$PULL" oci --checksum-file=yes foo/bar) |& grep -F "not supported for OCI images" >/dev/null
(! "$PULL" oci --checksum-file=no foo/bar) |& grep -F "not supported for OCI images" >/dev/null
+# … and so are target ranges, which only make sense for raw images
+(! "$PULL" tar --verify=no --direct --offset=4096 "file://$CFDIR/foo.tar" "$CFDIR/out") |& grep -F "not supported for tar images" >/dev/null
+(! "$PULL" tar --verify=no --size-max=4096 "file://$CFDIR/foo.tar" foo) |& grep -F "not supported for tar images" >/dev/null
+(! "$PULL" oci --direct --offset=4096 foo/bar "$CFDIR/out") |& grep -F "not supported for OCI images" >/dev/null
+(! "$PULL" oci --size-max=4096 foo/bar) |& grep -F "not supported for OCI images" >/dev/null
test ! -e "$CFDIR/out"
# A file whose name doesn't suggest it is a checksum file, which hence needs a checksum file covering it,