avcodec/cuviddec: map new NVDEC 4:2:0/4:2:2/4:4:4 output formats

The switch to the new NVDEC pixel formats made cuvid_derive_output_format()
and the format probe select AV_PIX_FMT_P012 for 12-bit 4:2:0,
AV_PIX_FMT_P210/P212 for 10/12-bit 4:2:2 and AV_PIX_FMT_YUV444P10MSB/
YUV444P12MSB for 10/12-bit 4:4:4, but cuvid_handle_video_sequence() still
only mapped the older formats to a CUVID surface format. Decoding such
streams therefore failed with "Unsupported output format" (AVERROR(EINVAL));
because every packet then errored without producing a frame, the failure
could also manifest as a hang rather than a clean error exit. The
system-memory copy path in cuvid_output_frame() rejected the same formats
with AVERROR_BUG.

Map P012 to cudaVideoSurfaceFormat_P016 (mirroring P010), P210/P212 to
cudaVideoSurfaceFormat_P216 (mirroring P216) and YUV444P10MSB/YUV444P12MSB
to cudaVideoSurfaceFormat_YUV444_16Bit (mirroring YUV444P16); the hardware
surface is physically 16-bit and the pixel format only records the number
of valid bits. Add the same formats to the system-memory copy path, whose
per-plane geometry is already derived generically from the pixel descriptor.

Signed-off-by: Diego de Souza <ddesouza@nvidia.com>
This commit is contained in:
Diego de Souza
2026-06-24 00:30:45 +02:00
committed by Timo Rothenpieler
parent 74db061f0a
commit a02dfb1262

View File

@@ -333,6 +333,7 @@ static int CUDAAPI cuvid_handle_video_sequence(void *opaque, CUVIDEOFORMAT* form
cuinfo.OutputFormat = cudaVideoSurfaceFormat_NV12;
break;
case AV_PIX_FMT_P010:
case AV_PIX_FMT_P012:
case AV_PIX_FMT_P016:
cuinfo.OutputFormat = cudaVideoSurfaceFormat_P016;
break;
@@ -340,6 +341,8 @@ static int CUDAAPI cuvid_handle_video_sequence(void *opaque, CUVIDEOFORMAT* form
case AV_PIX_FMT_NV16:
cuinfo.OutputFormat = cudaVideoSurfaceFormat_NV16;
break;
case AV_PIX_FMT_P210:
case AV_PIX_FMT_P212:
case AV_PIX_FMT_P216:
cuinfo.OutputFormat = cudaVideoSurfaceFormat_P216;
break;
@@ -347,6 +350,8 @@ static int CUDAAPI cuvid_handle_video_sequence(void *opaque, CUVIDEOFORMAT* form
case AV_PIX_FMT_YUV444P:
cuinfo.OutputFormat = cudaVideoSurfaceFormat_YUV444;
break;
case AV_PIX_FMT_YUV444P10MSB:
case AV_PIX_FMT_YUV444P12MSB:
case AV_PIX_FMT_YUV444P16:
cuinfo.OutputFormat = cudaVideoSurfaceFormat_YUV444_16Bit;
break;
@@ -632,12 +637,17 @@ static int cuvid_output_frame(AVCodecContext *avctx, AVFrame *frame)
}
} else if (avctx->pix_fmt == AV_PIX_FMT_NV12 ||
avctx->pix_fmt == AV_PIX_FMT_P010 ||
avctx->pix_fmt == AV_PIX_FMT_P012 ||
avctx->pix_fmt == AV_PIX_FMT_P016 ||
#ifdef NVDEC_HAVE_422_SUPPORT
avctx->pix_fmt == AV_PIX_FMT_NV16 ||
avctx->pix_fmt == AV_PIX_FMT_P210 ||
avctx->pix_fmt == AV_PIX_FMT_P212 ||
avctx->pix_fmt == AV_PIX_FMT_P216 ||
#endif
avctx->pix_fmt == AV_PIX_FMT_YUV444P ||
avctx->pix_fmt == AV_PIX_FMT_YUV444P10MSB ||
avctx->pix_fmt == AV_PIX_FMT_YUV444P12MSB ||
avctx->pix_fmt == AV_PIX_FMT_YUV444P16) {
unsigned int offset = 0;
AVFrame *tmp_frame = av_frame_alloc();