har0ke/FFmpeg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

avcodec/simple_idct_template: fix rounding of the special DC case for 10bit

Browse Source 
MSE doesnt change for the test as the code only triggers
for very sparse matrixes

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
This commit is contained in:
Michael Niedermayer 2014-01-12 02:27:00 +01:00
parent a7ea733b72
commit 4b0cad6596
3 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libavcodec/simple_idct_template.c
View File

@ -107,10 +107,10 @@ static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift)
#define ROW0_MASK (0xffffLL << 48 * HAVE_BIGENDIAN)
if (((((uint64_t *)row)[0] & ~ROW0_MASK) | ((uint64_t *)row)[1]) == 0) {
uint64_t temp;
if (DC_SHIFT - extra_shift > 0) {
if (DC_SHIFT - extra_shift >= 0) {
temp = (row[0] << (DC_SHIFT - extra_shift)) & 0xffff;
} else {
temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff;
temp = ((row[0] + (1<<(extra_shift - DC_SHIFT-1))) >> (extra_shift - DC_SHIFT)) & 0xffff;
}
temp += temp << 16;
temp += temp << 32;
@ -124,10 +124,10 @@ static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift)
((uint32_t*)row)[3] |
row[1])) {
uint32_t temp;
if (DC_SHIFT - extra_shift > 0) {
if (DC_SHIFT - extra_shift >= 0) {
temp = (row[0] << (DC_SHIFT - extra_shift)) & 0xffff;
} else {
temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff;
temp = ((row[0] + (1<<(extra_shift - DC_SHIFT-1))) >> (extra_shift - DC_SHIFT)) & 0xffff;
}
temp += temp << 16;
((uint32_t*)row)[0]=((uint32_t*)row)[1] =

4
tests/ref/fate/prores-422_lt
View File

@ -1,3 +1,3 @@
#tb 0: 100/2997
0, 0, 0, 1, 8294400, 0xdaff4019
0, 1, 1, 1, 8294400, 0xdaff4019
0, 0, 0, 1, 8294400, 0x4ca110c7
0, 1, 1, 1, 8294400, 0x4ca110c7

2
tests/ref/vsynth/vsynth2-prores
View File

@ -1,4 +1,4 @@
af885572d77d28e5ede27932bd68cdde *tests/data/fate/vsynth2-prores.mov
2844076 tests/data/fate/vsynth2-prores.mov
387a3c23586c679b411c3eff2c9d9d06 *tests/data/fate/vsynth2-prores.out.rawvideo
03fd29e3963716a09d232b6f817ecb57 *tests/data/fate/vsynth2-prores.out.rawvideo
stddev: 1.31 PSNR: 45.77 MAXDIFF: 11 bytes: 7603200/ 7603200