swscale/lut3d: simplify and hard-code 3DLUT format

The ops-based 3DLUT approach will take care of appropriately normalizing
the input to the expected domain; so the format choice no longer matters here
except for the lut3d_apply() function, which will only be used by the legacy
reference code path. So we can just continue hard-coding the format there.

Sponsored-by: Sovereign Tech Fund
Signed-off-by: Niklas Haas <git@haasn.dev>

libswscale/graph.c

@@ -702,8 +702,8 @@ static void run_lut3d(const SwsFrame *out, const SwsFrame *in, int y, int h,
     frame_shift(in,  y, in_data);
     frame_shift(out, y, out_data);
 
-    ff_sws_lut3d_apply(lut, in_data[0], in->linesize[0], out_data[0],
-                       out->linesize[0], out->width, h);
+    ff_sws_lut3d_apply_rgba64(lut, in_data[0], in->linesize[0], out_data[0],
+                              out->linesize[0], out->width, h);
 }
 
 static int adapt_colors(SwsGraph *graph, const SwsFormat *src_fmt,
@@ -712,7 +712,6 @@ static int adapt_colors(SwsGraph *graph, const SwsFormat *src_fmt,
 {
     SwsFormat src = *src_fmt;
     SwsFormat dst = *dst_fmt;
-    enum AVPixelFormat fmt_in, fmt_out;
     SwsColorMap map = {0};
     SwsLut3D *lut;
     int ret;
@@ -746,11 +745,16 @@ static int adapt_colors(SwsGraph *graph, const SwsFormat *src_fmt,
     if (!lut)
         return AVERROR(ENOMEM);
 
-    fmt_in  = ff_sws_lut3d_pick_pixfmt(&src, 0);
-    fmt_out = ff_sws_lut3d_pick_pixfmt(&dst, 1);
-    if (fmt_in != src.format) {
+    ret = ff_sws_lut3d_generate(lut, &map);
+    if (ret < 0) {
+        av_refstruct_unref(&lut);
+        return ret;
+    }
+
+    const enum AVPixelFormat fmt = AV_PIX_FMT_RGBA64;
+    if (src.format != fmt) {
         SwsFormat tmp = src;
-        tmp.format = fmt_in;
+        tmp.format = fmt;
         ret = add_convert_pass(graph, &src, &tmp, input, &input);
         if (ret < 0) {
             av_refstruct_unref(&lut);
@@ -758,13 +762,7 @@ static int adapt_colors(SwsGraph *graph, const SwsFormat *src_fmt,
         }
     }
 
-    ret = ff_sws_lut3d_generate(lut, fmt_in, fmt_out, &map);
-    if (ret < 0) {
-        av_refstruct_unref(&lut);
-        return ret;
-    }
-
-    return ff_sws_graph_add_pass(graph, fmt_out, src.width, src.height,
+    return ff_sws_graph_add_pass(graph, fmt, src.width, src.height,
                                  input, 0, 1, run_lut3d, setup_lut3d, lut,
                                  free_lut3d, output);
 }

libswscale/lut3d.c

@@ -42,16 +42,6 @@ SwsLut3D *ff_sws_lut3d_alloc(void)
     return lut3d;
 }
 
-bool ff_sws_lut3d_test_fmt(enum AVPixelFormat fmt, int output)
-{
-    return fmt == AV_PIX_FMT_RGBA64;
-}
-
-enum AVPixelFormat ff_sws_lut3d_pick_pixfmt(const SwsFormat *fmt, int output)
-{
-    return AV_PIX_FMT_RGBA64;
-}
-
 /**
  * v0 and v1 are 'black' and 'white'
  * v2 and v3 are closest RGB/CMY vertices
@@ -205,14 +195,10 @@ static av_always_inline v3u16_t apply_tone_map(const SwsLut3D *lut3d, v3u16_t ip
     return ipt;
 }
 
-int ff_sws_lut3d_generate(SwsLut3D *lut3d, enum AVPixelFormat fmt_in,
-                          enum AVPixelFormat fmt_out, const SwsColorMap *map)
+int ff_sws_lut3d_generate(SwsLut3D *lut3d, const SwsColorMap *map)
 {
     int ret;
 
-    if (!ff_sws_lut3d_test_fmt(fmt_in, 0) || !ff_sws_lut3d_test_fmt(fmt_out, 1))
-        return AVERROR(EINVAL);
-
     lut3d->dynamic = map->src.frame_peak.num > 0;
     lut3d->map = *map;
 
@@ -245,8 +231,9 @@ void ff_sws_lut3d_update(SwsLut3D *lut3d, const SwsColor *new_src)
     lut3d->tone_map[TONE_LUT_SIZE] = lut3d->tone_map[TONE_LUT_SIZE - 1];
 }
 
-void ff_sws_lut3d_apply(const SwsLut3D *lut3d, const uint8_t *in, int in_stride,
-                        uint8_t *out, int out_stride, int w, int h)
+void ff_sws_lut3d_apply_rgba64(const SwsLut3D *lut3d, const uint8_t *in,
+                               int in_stride, uint8_t *out, int out_stride,
+                               int w, int h)
 {
     while (h--) {
         const uint16_t *in16 = (const uint16_t *) in;

libswscale/lut3d.h

@@ -64,16 +64,6 @@ typedef struct SwsLut3D {
  */
 SwsLut3D *ff_sws_lut3d_alloc(void);
 
-/**
- * Test to see if a given format is supported by the 3DLUT input/output code.
- */
-bool ff_sws_lut3d_test_fmt(enum AVPixelFormat fmt, int output);
-
-/**
- * Pick the best compatible pixfmt for a given SwsFormat.
- */
-enum AVPixelFormat ff_sws_lut3d_pick_pixfmt(const SwsFormat *fmt, int output);
-
 /**
  * Recalculate the (static) 3DLUT state with new settings. This will recompute
  * everything. To only update per-frame tone mapping state, instead call
@@ -81,8 +71,7 @@ enum AVPixelFormat ff_sws_lut3d_pick_pixfmt(const SwsFormat *fmt, int output);
  *
  * Returns 0 or a negative error code.
  */
-int ff_sws_lut3d_generate(SwsLut3D *lut3d, enum AVPixelFormat fmt_in,
-                          enum AVPixelFormat fmt_out, const SwsColorMap *map);
+int ff_sws_lut3d_generate(SwsLut3D *lut3d, const SwsColorMap *map);
 
 /**
  * Update the tone mapping state. This will only use per-frame metadata. The
@@ -91,10 +80,10 @@ int ff_sws_lut3d_generate(SwsLut3D *lut3d, enum AVPixelFormat fmt_in,
 void ff_sws_lut3d_update(SwsLut3D *lut3d, const SwsColor *new_src);
 
 /**
- * Applies a color transformation to a plane. The format must match the format
- * provided during ff_sws_lut3d_update().
+ * Applies a color transformation to a plane in RGBA64 format.
  */
-void ff_sws_lut3d_apply(const SwsLut3D *lut3d, const uint8_t *in, int in_stride,
-                        uint8_t *out, int out_stride, int w, int h);
+void ff_sws_lut3d_apply_rgba64(const SwsLut3D *lut3d, const uint8_t *in,
+                               int in_stride, uint8_t *out, int out_stride,
+                               int w, int h);
 
 #endif /* SWSCALE_LUT3D_H */