har0ke/FFmpeg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

prores-idct: fix overflow in c code.

Browse Source 
Fix the fate ref for prores-422_proxy by reverting the changes to it
in commit f492df0927c42da174edb674857670fc50abc5dc

Signed-off-by: Janne Grunau <janne-libav@jannau.net>
This commit is contained in:
Ronald S. Bultje 2011-10-14 09:27:59 -07:00 committed by Janne Grunau
parent 8aa80ec5e3
commit f78cd0c243
3 changed files with 29 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
libavcodec/simple_idct.c
View File

@ -108,7 +108,7 @@ void ff_simple_idct248_put(uint8_t *dest, int line_size, DCTELEM *block)
/* IDCT8 on each line */ /* IDCT8 on each line */
for(i=0; i<8; i++) { for(i=0; i<8; i++) {
idctRowCondDC_8(block + i*8); idctRowCondDC_8(block + i*8, 0);
} }
/* IDCT4 and store */ /* IDCT4 and store */
@ -183,7 +183,7 @@ void ff_simple_idct84_add(uint8_t *dest, int line_size, DCTELEM *block)
/* IDCT8 on each line */ /* IDCT8 on each line */
for(i=0; i<4; i++) { for(i=0; i<4; i++) {
idctRowCondDC_8(block + i*8); idctRowCondDC_8(block + i*8, 0);
} }
/* IDCT4 and store */ /* IDCT4 and store */
@ -230,10 +230,7 @@ void ff_prores_idct(DCTELEM *block, const int16_t *qmat)
block[i] *= qmat[i]; block[i] *= qmat[i];
for (i = 0; i < 8; i++) for (i = 0; i < 8; i++)
idctRowCondDC_10(block + i*8); idctRowCondDC_10(block + i*8, 2);
for (i = 0; i < 64; i++)
block[i] >>= 2;
for (i = 0; i < 8; i++) for (i = 0; i < 8; i++)
idctSparseCol_10(block + i); idctSparseCol_10(block + i);

38
libavcodec/simple_idct_template.c
View File

@ -85,14 +85,19 @@
#endif #endif
static inline void FUNC(idctRowCondDC)(DCTELEM *row) static inline void FUNC(idctRowCondDC)(DCTELEM *row, int extra_shift)
{ {
int a0, a1, a2, a3, b0, b1, b2, b3; int a0, a1, a2, a3, b0, b1, b2, b3;
#if HAVE_FAST_64BIT #if HAVE_FAST_64BIT
#define ROW0_MASK (0xffffLL << 48 * HAVE_BIGENDIAN) #define ROW0_MASK (0xffffLL << 48 * HAVE_BIGENDIAN)
if (((((uint64_t *)row)[0] & ~ROW0_MASK) | ((uint64_t *)row)[1]) == 0) { if (((((uint64_t *)row)[0] & ~ROW0_MASK) | ((uint64_t *)row)[1]) == 0) {
uint64_t temp = (row[0] << DC_SHIFT) & 0xffff; uint64_t temp;
if (DC_SHIFT - extra_shift > 0) {
temp = (row[0] << (DC_SHIFT - extra_shift)) & 0xffff;
} else {
temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff;
}
temp += temp << 16; temp += temp << 16;
temp += temp << 32; temp += temp << 32;
((uint64_t *)row)[0] = temp; ((uint64_t *)row)[0] = temp;
@ -104,7 +109,12 @@ static inline void FUNC(idctRowCondDC)(DCTELEM *row)
((uint32_t*)row)[2] | ((uint32_t*)row)[2] |
((uint32_t*)row)[3] | ((uint32_t*)row)[3] |
row[1])) { row[1])) {
uint32_t temp = (row[0] << DC_SHIFT) & 0xffff; uint32_t temp;
if (DC_SHIFT - extra_shift > 0) {
temp = (row[0] << (DC_SHIFT - extra_shift)) & 0xffff;
} else {
temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff;
}
temp += temp << 16; temp += temp << 16;
((uint32_t*)row)[0]=((uint32_t*)row)[1] = ((uint32_t*)row)[0]=((uint32_t*)row)[1] =
((uint32_t*)row)[2]=((uint32_t*)row)[3] = temp; ((uint32_t*)row)[2]=((uint32_t*)row)[3] = temp;
@ -150,14 +160,14 @@ static inline void FUNC(idctRowCondDC)(DCTELEM *row)
MAC(b3, -W1, row[7]); MAC(b3, -W1, row[7]);
} }
row[0] = (a0 + b0) >> ROW_SHIFT; row[0] = (a0 + b0) >> (ROW_SHIFT + extra_shift);
row[7] = (a0 - b0) >> ROW_SHIFT; row[7] = (a0 - b0) >> (ROW_SHIFT + extra_shift);
row[1] = (a1 + b1) >> ROW_SHIFT; row[1] = (a1 + b1) >> (ROW_SHIFT + extra_shift);
row[6] = (a1 - b1) >> ROW_SHIFT; row[6] = (a1 - b1) >> (ROW_SHIFT + extra_shift);
row[2] = (a2 + b2) >> ROW_SHIFT; row[2] = (a2 + b2) >> (ROW_SHIFT + extra_shift);
row[5] = (a2 - b2) >> ROW_SHIFT; row[5] = (a2 - b2) >> (ROW_SHIFT + extra_shift);
row[3] = (a3 + b3) >> ROW_SHIFT; row[3] = (a3 + b3) >> (ROW_SHIFT + extra_shift);
row[4] = (a3 - b3) >> ROW_SHIFT; row[4] = (a3 - b3) >> (ROW_SHIFT + extra_shift);
} }
#define IDCT_COLS do { \ #define IDCT_COLS do { \
@ -284,7 +294,7 @@ void FUNC(ff_simple_idct_put)(uint8_t *dest_, int line_size, DCTELEM *block)
line_size /= sizeof(pixel); line_size /= sizeof(pixel);
for (i = 0; i < 8; i++) for (i = 0; i < 8; i++)
FUNC(idctRowCondDC)(block + i*8); FUNC(idctRowCondDC)(block + i*8, 0);
for (i = 0; i < 8; i++) for (i = 0; i < 8; i++)
FUNC(idctSparseColPut)(dest + i, line_size, block + i); FUNC(idctSparseColPut)(dest + i, line_size, block + i);
@ -298,7 +308,7 @@ void FUNC(ff_simple_idct_add)(uint8_t *dest_, int line_size, DCTELEM *block)
line_size /= sizeof(pixel); line_size /= sizeof(pixel);
for (i = 0; i < 8; i++) for (i = 0; i < 8; i++)
FUNC(idctRowCondDC)(block + i*8); FUNC(idctRowCondDC)(block + i*8, 0);
for (i = 0; i < 8; i++) for (i = 0; i < 8; i++)
FUNC(idctSparseColAdd)(dest + i, line_size, block + i); FUNC(idctSparseColAdd)(dest + i, line_size, block + i);
@ -309,7 +319,7 @@ void FUNC(ff_simple_idct)(DCTELEM *block)
int i; int i;
for (i = 0; i < 8; i++) for (i = 0; i < 8; i++)
FUNC(idctRowCondDC)(block + i*8); FUNC(idctRowCondDC)(block + i*8, 0);
for (i = 0; i < 8; i++) for (i = 0; i < 8; i++)
FUNC(idctSparseCol)(block + i); FUNC(idctSparseCol)(block + i);

4
tests/ref/fate/prores-422_proxy
View File

@ -1,2 +1,2 @@
0, 0, 8294400, 0x3099e361 0, 0, 8294400, 0x51d29320
0, 3003, 8294400, 0x3099e361 0, 3003, 8294400, 0x51d29320