From b96701098fd8bc33a7726b895ca92102da65202d Mon Sep 17 00:00:00 2001 From: Omkhar Arasaratnam Date: Mon, 6 Jul 2026 12:22:29 -0400 Subject: [PATCH] avformat/tls_mbedtls: check the certificate verification result in tls_handshake() tls_mbedtls verifies the peer certificate manually via mbedtls_ssl_get_verify_result() (it uses MBEDTLS_SSL_VERIFY_OPTIONAL), but only in tls_open(). On the external-socket path (external_sock=1) tls_open() skips the handshake; it then runs later through the url_handshake hook, tls_handshake(), which never checks the verification result, so with verify=1 an untrusted peer certificate would be accepted. The only in-tree user of this path is the WHIP muxer, which sets verify=0 (WebRTC binds the peer via the SDP a=fingerprint, not a CA), so this is not reachable today. This change is defence-in-depth: it makes tls_handshake() honor verify symmetrically with the existing tls_open() check. Signed-off-by: Omkhar Arasaratnam --- libavformat/tls_mbedtls.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/libavformat/tls_mbedtls.c b/libavformat/tls_mbedtls.c index bfa5103596..8f7ace5e63 100644 --- a/libavformat/tls_mbedtls.c +++ b/libavformat/tls_mbedtls.c @@ -475,6 +475,7 @@ static int tls_handshake(URLContext *h) TLSContext *tls_ctx = h->priv_data; TLSShared *shr = &tls_ctx->tls_shared; URLContext *uc = shr->is_dtls ? shr->udp : shr->tcp; + uint32_t verify_res_flags; int ret; uc->flags &= ~AVIO_FLAG_NONBLOCK; @@ -490,6 +491,18 @@ static int tls_handshake(URLContext *h) } } + if (shr->verify) { + // check the result of the certificate verification + if ((verify_res_flags = mbedtls_ssl_get_verify_result(&tls_ctx->ssl_context)) != 0) { + av_log(h, AV_LOG_ERROR, "mbedtls_ssl_get_verify_result reported problems "\ + "with the certificate verification, returned flags: %"PRIu32"\n", + verify_res_flags); + if (verify_res_flags & MBEDTLS_X509_BADCERT_NOT_TRUSTED) + av_log(h, AV_LOG_ERROR, "The certificate is not correctly signed by the trusted CA.\n"); + return AVERROR(EIO); + } + } + return ret; }