diff --git a/libswscale/ops_dispatch.c b/libswscale/ops_dispatch.c
index b79d801fff..8b04ef1c96 100644
--- a/libswscale/ops_dispatch.c
+++ b/libswscale/ops_dispatch.c
@@ -182,6 +182,7 @@ static int op_pass_setup(const SwsFrame *out, const SwsFrame *in,
 {
     const AVPixFmtDescriptor *indesc  = av_pix_fmt_desc_get(in->format);
     const AVPixFmtDescriptor *outdesc = av_pix_fmt_desc_get(out->format);
+    const bool float_in = indesc->flags & AV_PIX_FMT_FLAG_FLOAT;
 
     SwsOpPass *p = pass->priv;
     SwsOpExec *exec = &p->exec_base;
@@ -204,7 +205,15 @@ static int op_pass_setup(const SwsFrame *out, const SwsFrame *in,
         int chroma = idx == 1 || idx == 2;
         int sub_x  = chroma ? indesc->log2_chroma_w : 0;
         int sub_y  = chroma ? indesc->log2_chroma_h : 0;
-        size_t safe_bytes = safe_bytes_pad(in->linesize[idx], comp->over_read);
+
+        size_t input_bytes = in->linesize[idx];
+        if (p->filter_size_h && float_in) {
+            /* Floating point inputs may contain NaN / Infinity in the padding */
+            const int plane_w = AV_CEIL_RSHIFT(in->width, sub_x);
+            input_bytes = pixel_bytes(plane_w, p->pixel_bits_in, AV_ROUND_UP);
+        }
+
+        size_t safe_bytes = safe_bytes_pad(input_bytes, comp->over_read);
         size_t safe_blocks_in;
         if (exec->in_offset_x) {
             size_t filter_size = pixel_bytes(p->filter_size_h, p->pixel_bits_in,