har0ke/FFmpeg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

avcodec/fraps: Fix segfault with negative linesizes

Browse Source 
Using unsigned and negative linesizes doesn't really work.
Use ptrdiff_t instead. This fixes the fraps-v0 and fraps-v1
FATE tests with negative linesizes.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
This commit is contained in:
Andreas Rheinhardt 2022-09-30 18:32:33 +02:00
parent fcfa6965e5
commit 451b310d4f
1 changed files with 5 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
libavcodec/fraps.c
View File

@ -141,7 +141,6 @@ static int decode_frame(AVCodecContext *avctx, AVFrame *f,
int buf_size = avpkt->size; int buf_size = avpkt->size;
uint32_t header; uint32_t header;
unsigned int version,header_size; unsigned int version,header_size;
unsigned int x, y;
const uint32_t *buf32; const uint32_t *buf32;
uint32_t *luma1,*luma2,*cb,*cr; uint32_t *luma1,*luma2,*cb,*cr;
uint32_t offs[4]; uint32_t offs[4];
@ -238,12 +237,12 @@ static int decode_frame(AVCodecContext *avctx, AVFrame *f,
} }
buf32 = (const uint32_t*)buf; buf32 = (const uint32_t*)buf;
for (y = 0; y < avctx->height / 2; y++) { for (ptrdiff_t y = 0; y < avctx->height / 2; y++) {
luma1 = (uint32_t*)&f->data[0][ y * 2 * f->linesize[0] ]; luma1 = (uint32_t*)&f->data[0][ y * 2 * f->linesize[0] ];
luma2 = (uint32_t*)&f->data[0][ (y * 2 + 1) * f->linesize[0] ]; luma2 = (uint32_t*)&f->data[0][ (y * 2 + 1) * f->linesize[0] ];
cr = (uint32_t*)&f->data[1][ y * f->linesize[1] ]; cr = (uint32_t*)&f->data[1][ y * f->linesize[1] ];
cb = (uint32_t*)&f->data[2][ y * f->linesize[2] ]; cb = (uint32_t*)&f->data[2][ y * f->linesize[2] ];
for (x = 0; x < avctx->width; x += 8) { for (ptrdiff_t x = 0; x < avctx->width; x += 8) {
*luma1++ = *buf32++; *luma1++ = *buf32++;
*luma1++ = *buf32++; *luma1++ = *buf32++;
*luma2++ = *buf32++; *luma2++ = *buf32++;
@ -258,18 +257,18 @@ static int decode_frame(AVCodecContext *avctx, AVFrame *f,
if (is_pal) { if (is_pal) {
uint32_t *pal = (uint32_t *)f->data[1]; uint32_t *pal = (uint32_t *)f->data[1];
for (y = 0; y < 256; y++) { for (unsigned y = 0; y < 256; y++) {
pal[y] = AV_RL32(buf) | 0xFF000000; pal[y] = AV_RL32(buf) | 0xFF000000;
buf += 4; buf += 4;
} }
for (y = 0; y <avctx->height; y++) for (ptrdiff_t y = 0; y < avctx->height; y++)
memcpy(&f->data[0][y * f->linesize[0]], memcpy(&f->data[0][y * f->linesize[0]],
&buf[y * avctx->width], &buf[y * avctx->width],
avctx->width); avctx->width);
} else { } else {
/* Fraps v1 is an upside-down BGR24 */ /* Fraps v1 is an upside-down BGR24 */
for (y = 0; y<avctx->height; y++) for (ptrdiff_t y = 0; y < avctx->height; y++)
memcpy(&f->data[0][(avctx->height - y - 1) * f->linesize[0]], memcpy(&f->data[0][(avctx->height - y - 1) * f->linesize[0]],
&buf[y * avctx->width * 3], &buf[y * avctx->width * 3],
3 * avctx->width); 3 * avctx->width);