swscale/ops: keep track of correct dither min/max

Mostly, this just affects the metadata in benign ways, e.g.:

 rgb24 -> yuv444p:
   [ u8 +++X] SWS_OP_READ         : 3 elem(s) packed >> 0
     min: {0, 0, 0, _}, max: {255, 255, 255, _}
   [ u8 +++X] SWS_OP_CONVERT      : u8 -> f32
     min: {0, 0, 0, _}, max: {255, 255, 255, _}
   [f32 ...X] SWS_OP_LINEAR       : matrix3+off3 [...]
     min: {16, 16, 16, _}, max: {235, 240, 240, _}
   [f32 ...X] SWS_OP_DITHER       : 16x16 matrix + {0 3 2 -1}
-    min: {33/2, 33/2, 33/2, _}, max: {471/2, 481/2, 481/2, _}
+    min: {16.001953, 16.001953, 16.001953, _}, max: {235.998047, 240.998047, 240.998047, _}
   [f32 +++X] SWS_OP_CONVERT      : f32 -> u8
     min: {16, 16, 16, _}, max: {235, 240, 240, _}
   [ u8 XXXX] SWS_OP_WRITE        : 3 elem(s) planar >> 0
     (X = unused, z = byteswapped, + = exact, 0 = zero)

However, it surprisingly actually includes a semantic change, whenever
converting from limited range to monob or monow:

 yuv444p -> monow:
   [ u8 +XXX] SWS_OP_READ         : 1 elem(s) planar >> 0
     min: {0, _, _, _}, max: {255, _, _, _}
   [ u8 +XXX] SWS_OP_CONVERT      : u8 -> f32
     min: {0, _, _, _}, max: {255, _, _, _}
   [f32 .XXX] SWS_OP_LINEAR       : luma [...]
     min: {-20/219, _, _, _}, max: {235/219, _, _, _}
   [f32 .XXX] SWS_OP_DITHER       : 16x16 matrix + {0 -1 -1 -1}
-    min: {179/438, _, _, _}, max: {689/438, _, _, _}
+    min: {-0.089371, _, _, _}, max: {2.071106, _, _, _}
+  [f32 .XXX] SWS_OP_MAX          : {0 0 0 0} <= x
+    min: {0, _, _, _}, max: {2.071106, _, _, _}
   [f32 .XXX] SWS_OP_MIN          : x <= {1 _ _ _}
-    min: {179/438, _, _, _}, max: {1, _, _, _}
+    min: {0, _, _, _}, max: {1, _, _, _}
   [f32 +XXX] SWS_OP_CONVERT      : f32 -> u8
     min: {0, _, _, _}, max: {1, _, _, _}
   [ u8 XXXX] SWS_OP_WRITE        : 1 elem(s) planar >> 3
     (X = unused, z = byteswapped, + = exact, 0 = zero)

Note the presence of an extra SWS_OP_MAX, to correctly clamp sub-blacks
(values below 16) to 0.0, rather than underflowing. This was previously
undetected because the dither was modelled as adding 0.5 to every pixel value,
but that's only true on average - not always.

Signed-off-by: Niklas Haas <git@haasn.dev>

libswscale/ops.c

@@ -304,6 +304,7 @@ void ff_sws_op_list_update_comps(SwsOpList *ops)
         switch (op->op) {
         case SWS_OP_READ:
         case SWS_OP_LINEAR:
+        case SWS_OP_DITHER:
         case SWS_OP_SWAP_BYTES:
         case SWS_OP_UNPACK:
         case SWS_OP_FILTER_H:
@@ -364,9 +365,13 @@ void ff_sws_op_list_update_comps(SwsOpList *ops)
         case SWS_OP_DITHER:
             /* Strip zero flag because of the nonzero dithering offset */
             for (int i = 0; i < 4; i++) {
+                op->comps.min[i] = prev.min[i];
+                op->comps.max[i] = prev.max[i];
                 if (op->dither.y_offset[i] < 0)
                     continue;
                 op->comps.flags[i] = prev.flags[i] & ~SWS_COMP_ZERO;
+                op->comps.min[i] = av_add_q(op->comps.min[i], op->dither.min);
+                op->comps.max[i] = av_add_q(op->comps.max[i], op->dither.max);
             }
             break;
         case SWS_OP_UNPACK:

tests/ref/fate/sws-ops-list

@@ -1 +1 @@
-a2ed0581163448a2c398ee9992e8eaf6
+374319dfd2b74cb5b69dac68b627fa9b