avformat/tls_mbedtls: check the certificate verification result in tls_handshake()

tls_mbedtls verifies the peer certificate manually via
mbedtls_ssl_get_verify_result() (it uses MBEDTLS_SSL_VERIFY_OPTIONAL), but
only in tls_open(). On the external-socket path (external_sock=1) tls_open()
skips the handshake; it then runs later through the url_handshake hook,
tls_handshake(), which never checks the verification result, so with verify=1
an untrusted peer certificate would be accepted.

The only in-tree user of this path is the WHIP muxer, which sets verify=0
(WebRTC binds the peer via the SDP a=fingerprint, not a CA), so this is not
reachable today. This change is defence-in-depth: it makes tls_handshake()
honor verify symmetrically with the existing tls_open() check.

Signed-off-by: Omkhar Arasaratnam <omkhar@linkedin.com>
(cherry picked from commit b96701098f)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Omkhar Arasaratnam
2026-07-06 12:22:29 -04:00
committed by Michael Niedermayer
parent 1b8b99dd45
commit f8051f532c

View File

@@ -475,6 +475,7 @@ static int tls_handshake(URLContext *h)
TLSContext *tls_ctx = h->priv_data;
TLSShared *shr = &tls_ctx->tls_shared;
URLContext *uc = shr->is_dtls ? shr->udp : shr->tcp;
uint32_t verify_res_flags;
int ret;
uc->flags &= ~AVIO_FLAG_NONBLOCK;
@@ -490,6 +491,18 @@ static int tls_handshake(URLContext *h)
}
}
if (shr->verify) {
// check the result of the certificate verification
if ((verify_res_flags = mbedtls_ssl_get_verify_result(&tls_ctx->ssl_context)) != 0) {
av_log(h, AV_LOG_ERROR, "mbedtls_ssl_get_verify_result reported problems "\
"with the certificate verification, returned flags: %"PRIu32"\n",
verify_res_flags);
if (verify_res_flags & MBEDTLS_X509_BADCERT_NOT_TRUSTED)
av_log(h, AV_LOG_ERROR, "The certificate is not correctly signed by the trusted CA.\n");
return AVERROR(EIO);
}
}
return ret;
}