From 435b74c6a5d4e457d9526aa21e7a6e68926bf52c Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sun, 12 May 2024 00:13:58 +0200 Subject: [PATCH] avcodec/mpegvideo_enc: Fix potential overflow in RD Fixes: CID1500285 Unintentional integer overflow Sponsored-by: Sovereign Tech Fund Signed-off-by: Michael Niedermayer (cherry picked from commit b6b2b01025e016ce29e5add57305384a663edcfc) Signed-off-by: Michael Niedermayer --- libavcodec/mpegvideo_enc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c index 9d048e3dec..c4c174a02e 100644 --- a/libavcodec/mpegvideo_enc.c +++ b/libavcodec/mpegvideo_enc.c @@ -1433,7 +1433,7 @@ static int estimate_best_b_count(MpegEncContext *s) goto fail; } - rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3); + rd += (out_size * (uint64_t)lambda2) >> (FF_LAMBDA_SHIFT - 3); } /* get the delayed frames */ @@ -1442,7 +1442,7 @@ static int estimate_best_b_count(MpegEncContext *s) ret = out_size; goto fail; } - rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3); + rd += (out_size * (uint64_t)lambda2) >> (FF_LAMBDA_SHIFT - 3); rd += c->error[0] + c->error[1] + c->error[2];