avcodec/lcevcdec: check the picture LCEVCdec will output against the one allocated

A frame that cannot be output is skipped and its base freed.

Fixes: out of array write
Fixes: yOAdvErxaeGg
Fixes: DAHozZoXi03N/hijack2_twogc.h264

Found-by: Ayoub Nabil Boubagrat <237098474+ayoubnabil@users.noreply.github.com>

Out of array write Replicated through UnModified FFmpeg with ASAN

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Michael Niedermayer authored and michaelni committed 2026-10-28 21:08:03 +00:00
1 parent 902df73ba8
commit fd6dfa613a
2 files changed
+49 -26

No files matched your search

Vendored
+1 -1
View File
@@ -7269,7 +7269,7 @@ enabled libklvanc && require libklvanc libklvanc/vanc.h klvanc_context_c
enabled libkvazaar && require_pkg_config libkvazaar "kvazaar >= 2.0.0" kvazaar.h kvz_api_get
enabled liblc3 && require_pkg_config liblc3 "lc3 >= 1.1.0" lc3.h lc3_hr_setup_encoder
enabled liblensfun && require_pkg_config liblensfun lensfun lensfun.h lf_db_create
enabled liblcevc_dec && require_pkg_config liblcevc_dec "lcevc_dec >= 4.0.0" "LCEVC/lcevc_dec.h" LCEVC_CreateDecoder
enabled liblcevc_dec && require_pkg_config liblcevc_dec "lcevc_dec >= 4.2.0" "LCEVC/lcevc_dec.h" LCEVC_CreateDecoder
if enabled libmfx && enabled libvpl; then
die "ERROR: can not use libmfx and libvpl together"
+48 -25
View File
@@ -75,34 +75,33 @@ static LCEVC_ColorFormat map_format(int format)
return LCEVC_ColorFormat_Unknown;
}
static int alloc_base_frame(FFLCEVCContext *lcevc,
const AVFrame *frame, LCEVC_PictureHandle *picture)
static int alloc_base_frame(FFLCEVCContext *lcevc, const AVFrame *frame,
LCEVC_PictureDesc *desc, LCEVC_PictureHandle *picture)
{
LCEVC_PictureDesc desc;
LCEVC_ColorFormat fmt = map_format(frame->format);
LCEVC_PicturePlaneDesc planes[AV_VIDEO_MAX_PLANES] = { 0 };
int width = frame->width - frame->crop_left - frame->crop_right;
int height = frame->height - frame->crop_top - frame->crop_bottom;
LCEVC_ReturnCode res;
res = LCEVC_DefaultPictureDesc(&desc, fmt, width, height);
res = LCEVC_DefaultPictureDesc(desc, fmt, width, height);
if (res != LCEVC_Success)
return AVERROR_EXTERNAL;
desc.cropTop = frame->crop_top;
desc.cropBottom = frame->crop_bottom;
desc.cropLeft = frame->crop_left;
desc.cropRight = frame->crop_right;
desc.sampleAspectRatioNum = frame->sample_aspect_ratio.num;
desc.sampleAspectRatioDen = frame->sample_aspect_ratio.den;
desc.colorRange = LCEVC_ColorRange_Unknown;
desc->cropTop = frame->crop_top;
desc->cropBottom = frame->crop_bottom;
desc->cropLeft = frame->crop_left;
desc->cropRight = frame->crop_right;
desc->sampleAspectRatioNum = frame->sample_aspect_ratio.num;
desc->sampleAspectRatioDen = frame->sample_aspect_ratio.den;
desc->colorRange = LCEVC_ColorRange_Unknown;
if (frame->color_range == AVCOL_RANGE_JPEG)
desc.colorRange = LCEVC_ColorRange_Full;
desc->colorRange = LCEVC_ColorRange_Full;
else if (frame->color_range == AVCOL_RANGE_MPEG)
desc.colorRange = LCEVC_ColorRange_Limited;
desc.colorPrimaries = (LCEVC_ColorPrimaries)frame->color_primaries;
desc.matrixCoefficients = (LCEVC_MatrixCoefficients)frame->colorspace;
desc.transferCharacteristics = (LCEVC_TransferCharacteristics)frame->color_trc;
desc->colorRange = LCEVC_ColorRange_Limited;
desc->colorPrimaries = (LCEVC_ColorPrimaries)frame->color_primaries;
desc->matrixCoefficients = (LCEVC_MatrixCoefficients)frame->colorspace;
desc->transferCharacteristics = (LCEVC_TransferCharacteristics)frame->color_trc;
for (int i = 0; i < AV_VIDEO_MAX_PLANES; i++) {
planes[i].firstSample = frame->data[i];
@@ -110,7 +109,7 @@ static int alloc_base_frame(FFLCEVCContext *lcevc,
}
/* Allocate LCEVC Picture */
res = LCEVC_AllocPictureExternal(lcevc->decoder, &desc, NULL, planes, picture);
res = LCEVC_AllocPictureExternal(lcevc->decoder, desc, NULL, planes, picture);
if (res != LCEVC_Success) {
return AVERROR_EXTERNAL;
}
@@ -177,10 +176,11 @@ static int lcevc_send_frame(FFLCEVCFrame *frame_ctx, const AVFrame *in)
const AVFrameSideData *sd = av_frame_get_side_data(in, AV_FRAME_DATA_LCEVC);
AVFrame *opaque;
LCEVC_PictureHandle picture;
LCEVC_PictureDesc desc, base_desc;
LCEVC_ReturnCode res;
int ret;
if (!sd || fmt == LCEVC_ColorFormat_Unknown)
if (!sd || fmt == LCEVC_ColorFormat_Unknown || lcevc->passthrough_mode == 1)
return 1;
res = LCEVC_SendDecoderEnhancementData(lcevc->decoder, lcevc->last_pts,
@@ -190,15 +190,15 @@ static int lcevc_send_frame(FFLCEVCFrame *frame_ctx, const AVFrame *in)
goto end;
}
ret = alloc_base_frame(lcevc, in, &picture);
ret = alloc_base_frame(lcevc, in, &base_desc, &picture);
if (ret < 0)
goto end;
goto fail;
opaque = av_frame_clone(in);
if (!opaque) {
LCEVC_FreePicture(lcevc->decoder, picture);
ret = AVERROR(ENOMEM);
goto end;
goto fail;
}
res = LCEVC_SetPictureUserData(lcevc->decoder, picture, opaque);
@@ -206,7 +206,7 @@ static int lcevc_send_frame(FFLCEVCFrame *frame_ctx, const AVFrame *in)
LCEVC_FreePicture(lcevc->decoder, picture);
av_frame_free(&opaque);
ret = AVERROR_EXTERNAL;
goto end;
goto fail;
}
res = LCEVC_SendDecoderBase(lcevc->decoder, lcevc->last_pts, picture, -1, opaque);
@@ -214,22 +214,45 @@ static int lcevc_send_frame(FFLCEVCFrame *frame_ctx, const AVFrame *in)
LCEVC_FreePicture(lcevc->decoder, picture);
av_frame_free(&opaque);
ret = AVERROR_EXTERNAL;
goto end;
goto fail;
}
res = LCEVC_PeekDecoderPictureDesc(lcevc->decoder, lcevc->last_pts, &desc);
if (res != LCEVC_Success ||
desc.width != frame_ctx->frame->width ||
desc.height != frame_ctx->frame->height ||
desc.colorFormat != map_format(frame_ctx->frame->format)) {
if (res == LCEVC_Success &&
desc.width == base_desc.width &&
desc.height == base_desc.height &&
desc.colorFormat == base_desc.colorFormat) {
av_log(lcevc, AV_LOG_VERBOSE, "Passing the base picture through\n");
ret = 1;
goto fail;
}
av_log(lcevc, AV_LOG_ERROR, "Output picture does not match the enhancement data\n");
ret = AVERROR_INVALIDDATA;
goto fail;
}
memset(&picture, 0, sizeof(picture));
ret = alloc_enhanced_frame(frame_ctx, &picture);
if (ret < 0)
goto end;
goto fail;
res = LCEVC_SendDecoderPicture(lcevc->decoder, picture);
if (res != LCEVC_Success) {
LCEVC_FreePicture(lcevc->decoder, picture);
ret = AVERROR_EXTERNAL;
goto end;
goto fail;
}
ret = 0;
goto end;
fail:
LCEVC_SkipDecoder(lcevc->decoder, lcevc->last_pts);
LCEVC_SynchronizeDecoder(lcevc->decoder, 0);
lcevc_flush_pictures(lcevc);
end:
lcevc->last_pts++;