swscale/cms: fix IPT rounding error

av_round16f() was incorrectly scaling to 65534, which would correspond to
a neutral PT offset of 32767 = (1 << 15) - 1, but the code was assuming a
value of (1 << 15). To fix it, and make the PT channel correctly symmetric
around the intended neutral value, we have to map PT = +0.5 to 65536, an
unrepresentable value. This is not an issue because the PT channel values are
strictly inside some subset of [-0.5, 0.5] in practice, for real in-gamut
color values - it's already an envelope that includes quite a bit of safety
margin.

Similarly, av_round16f() is also the wrong tool for the I/RGB channels,
because it incorrectly scaled those to 65534, an off by one of the intended
unorm16 full range peak of 65535. This silently resulted in e.g. RGBA64
true white (65535) not round-tripping through the 3DLUT.

Fix both by splitting this helper into two separate helpers, each of them
fixed to the correct, intended value range.

Sponsored-by: Sovereign Tech Fund
Signed-off-by: Niklas Haas <git@haasn.dev>
This commit is contained in:
Niklas Haas
2026-07-27 13:51:10 +02:00
committed by Niklas Haas
parent e43517c1d1
commit 96e0967eca

View File

@@ -562,9 +562,14 @@ static IPT saturation(const CmsCtx * ctx, IPT ipt)
return rgb2ipt(rgb, ctx->dst.content2lms);
}
static av_always_inline av_const uint16_t av_round16f(float x)
static av_always_inline av_const uint16_t round_unorm16(float x)
{
return av_clip_uint16(x * (UINT16_MAX - 1) + 0.5f);
return av_clip_uint16(x * UINT16_MAX + 0.5f);
}
static av_always_inline av_const uint16_t round_pt16(float x)
{
return av_clip_uint16(x * (1 << 16) + 0.5f);
}
/* Call this whenever the hue changes inside the loop body */
@@ -602,7 +607,6 @@ static void generate_slice(void *priv, int jobnr, int threadnr, int nb_jobs,
const float I_scale = 1.0f / (ctx.src.Imax - ctx.src.Imin);
const float I_offset = -ctx.src.Imin * I_scale;
const float PT_offset = (float) (1 << 15) / (UINT16_MAX - 1);
const float input_scale = 1.0f / (ctx.size_input - 1);
const float output_scale_PT = 1.0f / (ctx.size_output_PT - 1);
@@ -625,9 +629,9 @@ static void generate_slice(void *priv, int jobnr, int threadnr, int nb_jobs,
if (output) {
/* Save intermediate value to 3DLUT */
*input++ = (v3u16_t) {
av_round16f(I_scale * ipt.I + I_offset),
av_round16f(ipt.P + PT_offset),
av_round16f(ipt.T + PT_offset),
round_unorm16(I_scale * ipt.I + I_offset),
round_pt16(ipt.P + 0.5f),
round_pt16(ipt.T + 0.5f),
};
} else {
update_hue_peaks(&ctx, ipt.P, ipt.T);
@@ -641,9 +645,9 @@ static void generate_slice(void *priv, int jobnr, int threadnr, int nb_jobs,
c[2] = rgb.B;
ctx.dst.eotf_inv(ctx.dst.Lw, ctx.dst.Lb, c);
*input++ = (v3u16_t) {
av_round16f(c[0]),
av_round16f(c[1]),
av_round16f(c[2]),
round_unorm16(c[0]),
round_unorm16(c[1]),
round_unorm16(c[2]),
};
}
}
@@ -655,9 +659,9 @@ static void generate_slice(void *priv, int jobnr, int threadnr, int nb_jobs,
/* Generate split gamut mapping LUT */
for (int Tx = output_start; Tx < output_end; Tx++) {
const float T = output_scale_PT * Tx - PT_offset;
const float T = output_scale_PT * Tx - 0.5f;
for (int Px = 0; Px < ctx.size_output_PT; Px++) {
const float P = output_scale_PT * Px - PT_offset;
const float P = output_scale_PT * Px - 0.5f;
update_hue_peaks(&ctx, P, T);
for (int Ix = 0; Ix < ctx.size_output_I; Ix++) {
@@ -667,9 +671,9 @@ static void generate_slice(void *priv, int jobnr, int threadnr, int nb_jobs,
double c[3] = { rgb.R, rgb.G, rgb.B };
ctx.dst.eotf_inv(ctx.dst.Lw, ctx.dst.Lb, c);
*output++ = (v3u16_t) {
av_round16f(c[0]),
av_round16f(c[1]),
av_round16f(c[2]),
round_unorm16(c[0]),
round_unorm16(c[1]),
round_unorm16(c[2]),
};
}
}
@@ -759,7 +763,7 @@ void ff_sws_tone_map_generate(v2u16_t *lut, int size, const SwsColorMap *map)
const float I = src_scale * i + src_offset;
IPT ipt = tone_map_apply(&ctx, (IPT) { I, 1.0f });
lut[i] = (v2u16_t) {
av_round16f(dst_scale * ipt.I + dst_offset),
round_unorm16(dst_scale * ipt.I + dst_offset),
av_clip_uint16(ipt.P * (1 << 15) + 0.5f),
};
}