vulkan_encode: fix leak and swallowed errors in init_base_units()

The encoded session parameters buffer was never freed, and both
GetEncodedVideoSessionParametersKHR error paths returned err, which
is always 0 at that point, reporting success on failure.

(cherry picked from commit b672ae3905)
Co-Authored-by: Fable 5
This commit is contained in:
Lynne
2026-07-14 00:29:22 +09:00
committed by Michael Niedermayer
parent eebd95977f
commit 439ac0cb4b
3 changed files with 27 additions and 21 deletions

View File

@@ -1023,7 +1023,7 @@ static int init_base_units(AVCodecContext *avctx)
} else {
av_log(avctx, AV_LOG_ERROR, "Unable to get feedback for AV1 sequence header = %zu\n",
data_size);
return err;
return AVERROR_EXTERNAL;
}
ret = vk->GetEncodedVideoSessionParametersKHR(s->hwctx->act_dev, &params_info,
@@ -1031,7 +1031,8 @@ static int init_base_units(AVCodecContext *avctx)
&data_size, data);
if (ret != VK_SUCCESS) {
av_log(avctx, AV_LOG_ERROR, "Error writing feedback units\n");
return err;
err = AVERROR_EXTERNAL;
goto end;
}
av_log(avctx, AV_LOG_VERBOSE, "Feedback units written, overrides: %i\n",
@@ -1040,20 +1041,21 @@ static int init_base_units(AVCodecContext *avctx)
params_feedback.hasOverrides = 1;
/* No need to sync any overrides */
err = 0;
if (!params_feedback.hasOverrides)
return 0;
goto end;
/* Parse back tne units and override */
err = parse_feedback_units(avctx, data, data_size);
if (err < 0)
return err;
goto end;
/* Create final session parameters */
err = create_session_params(avctx);
if (err < 0)
return err;
return 0;
end:
av_free(data);
return err;
}
static int vulkan_encode_av1_add_obu(AVCodecContext *avctx,

View File

@@ -1148,7 +1148,7 @@ static int init_base_units(AVCodecContext *avctx)
return AVERROR(ENOMEM);
} else {
av_log(avctx, AV_LOG_ERROR, "Unable to get feedback for H.264 units = %zu\n", data_size);
return err;
return AVERROR_EXTERNAL;
}
ret = vk->GetEncodedVideoSessionParametersKHR(s->hwctx->act_dev, &params_info,
@@ -1156,7 +1156,8 @@ static int init_base_units(AVCodecContext *avctx)
&data_size, data);
if (ret != VK_SUCCESS) {
av_log(avctx, AV_LOG_ERROR, "Error writing feedback units\n");
return err;
err = AVERROR_EXTERNAL;
goto end;
}
av_log(avctx, AV_LOG_VERBOSE, "Feedback units written, overrides: %i (SPS: %i PPS: %i)\n",
@@ -1168,22 +1169,23 @@ static int init_base_units(AVCodecContext *avctx)
h264_params_feedback.hasStdPPSOverrides = 1;
/* No need to sync any overrides */
err = 0;
if (!params_feedback.hasOverrides)
return 0;
goto end;
/* Parse back tne units and override */
err = parse_feedback_units(avctx, data, data_size,
h264_params_feedback.hasStdSPSOverrides,
h264_params_feedback.hasStdPPSOverrides);
if (err < 0)
return err;
goto end;
/* Create final session parameters */
err = create_session_params(avctx);
if (err < 0)
return err;
return 0;
end:
av_free(data);
return err;
}
static int vulkan_encode_h264_add_nal(AVCodecContext *avctx,

View File

@@ -1317,7 +1317,7 @@ static int init_base_units(AVCodecContext *avctx)
return AVERROR(ENOMEM);
} else {
av_log(avctx, AV_LOG_ERROR, "Unable to get feedback for H.265 units = %zu\n", data_size);
return err;
return AVERROR_EXTERNAL;
}
ret = vk->GetEncodedVideoSessionParametersKHR(s->hwctx->act_dev, &params_info,
@@ -1325,7 +1325,8 @@ static int init_base_units(AVCodecContext *avctx)
&data_size, data);
if (ret != VK_SUCCESS) {
av_log(avctx, AV_LOG_ERROR, "Error writing feedback units\n");
return err;
err = AVERROR_EXTERNAL;
goto end;
}
av_log(avctx, AV_LOG_VERBOSE, "Feedback units written, overrides: %i (SPS: %i PPS: %i VPS: %i)\n",
@@ -1339,22 +1340,23 @@ static int init_base_units(AVCodecContext *avctx)
h265_params_feedback.hasStdPPSOverrides = 1;
/* No need to sync any overrides */
err = 0;
if (!params_feedback.hasOverrides)
return 0;
goto end;
/* Parse back tne units and override */
err = parse_feedback_units(avctx, data, data_size,
h265_params_feedback.hasStdSPSOverrides,
h265_params_feedback.hasStdPPSOverrides);
if (err < 0)
return err;
goto end;
/* Create final session parameters */
err = create_session_params(avctx);
if (err < 0)
return err;
return 0;
end:
av_free(data);
return err;
}
static int vulkan_encode_h265_add_nal(AVCodecContext *avctx,