From d94c293e6292d04cd87ab9c9ff861e3df8614bf5 Mon Sep 17 00:00:00 2001 From: Niklas Haas Date: Wed, 20 May 2026 17:53:56 +0200 Subject: [PATCH] swscale/ops_dispatch: prevent float over-read when horizontal filtering The code made the fundamental assumption that over-read into the padding bytes is okay to do; because the most that can happen is that those pixel values end up corrupted, which doesn't affect any adjacent pixels. However, this is not true for SWS_OP_FILTER_H, because this operation fundamentally mixes together horizontal pixels. Normally, this was fine, because the filter weights for those pixels are set to 0, and 0 * x = 0. However, that is not true for floating point inputs, which can contain Infinity; and 0 * Infinity = NaN, thus corrupting the entire pixel. Solve it by specifically preventing over-read when it would be unsafe. Signed-off-by: Niklas Haas --- libswscale/ops_dispatch.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/libswscale/ops_dispatch.c b/libswscale/ops_dispatch.c index b79d801fff..8b04ef1c96 100644 --- a/libswscale/ops_dispatch.c +++ b/libswscale/ops_dispatch.c @@ -182,6 +182,7 @@ static int op_pass_setup(const SwsFrame *out, const SwsFrame *in, { const AVPixFmtDescriptor *indesc = av_pix_fmt_desc_get(in->format); const AVPixFmtDescriptor *outdesc = av_pix_fmt_desc_get(out->format); + const bool float_in = indesc->flags & AV_PIX_FMT_FLAG_FLOAT; SwsOpPass *p = pass->priv; SwsOpExec *exec = &p->exec_base; @@ -204,7 +205,15 @@ static int op_pass_setup(const SwsFrame *out, const SwsFrame *in, int chroma = idx == 1 || idx == 2; int sub_x = chroma ? indesc->log2_chroma_w : 0; int sub_y = chroma ? indesc->log2_chroma_h : 0; - size_t safe_bytes = safe_bytes_pad(in->linesize[idx], comp->over_read); + + size_t input_bytes = in->linesize[idx]; + if (p->filter_size_h && float_in) { + /* Floating point inputs may contain NaN / Infinity in the padding */ + const int plane_w = AV_CEIL_RSHIFT(in->width, sub_x); + input_bytes = pixel_bytes(plane_w, p->pixel_bits_in, AV_ROUND_UP); + } + + size_t safe_bytes = safe_bytes_pad(input_bytes, comp->over_read); size_t safe_blocks_in; if (exec->in_offset_x) { size_t filter_size = pixel_bytes(p->filter_size_h, p->pixel_bits_in,