har0ke/FFmpeg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

avcodec/motion_est: Fix invalid left shift of negative numbers

Browse Source 
Affected many FATE-tests.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
(cherry picked from commit 3ef65fd4d1d0cdfa0b60351c719bef93d3664ea2)
This commit is contained in:
Andreas Rheinhardt 2021-03-26 18:18:05 +01:00 committed by Andreas Rheinhardt
parent cc3b05e424
commit b4b2f88cab
2 changed files with 32 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
libavcodec/motion_est.c
View File

@ -109,8 +109,8 @@ static av_always_inline int cmp_direct_inline(MpegEncContext *s, const int x, co
me_cmp_func cmp_func, me_cmp_func chroma_cmp_func, int qpel){
MotionEstContext * const c= &s->me;
const int stride= c->stride;
const int hx= subx + (x<<(1+qpel));
const int hy= suby + (y<<(1+qpel));
const int hx = subx + x * (1 << (1 + qpel));
const int hy = suby + y * (1 << (1 + qpel));
uint8_t * const * const ref= c->ref[ref_index];
uint8_t * const * const src= c->src[src_index];
int d;
@ -599,7 +599,7 @@ static inline int h263_mv4_search(MpegEncContext *s, int mx, int my, int shift)
P_LEFT[0] = s->current_picture.motion_val[0][mot_xy - 1][0];
P_LEFT[1] = s->current_picture.motion_val[0][mot_xy - 1][1];
if(P_LEFT[0] > (c->xmax<<shift)) P_LEFT[0] = (c->xmax<<shift);
if (P_LEFT[0] > c->xmax * (1 << shift)) P_LEFT[0] = c->xmax * (1 << shift);
/* special case for first line */
if (s->first_slice_line && block<2) {
@ -610,10 +610,10 @@ static inline int h263_mv4_search(MpegEncContext *s, int mx, int my, int shift)
P_TOP[1] = s->current_picture.motion_val[0][mot_xy - mot_stride ][1];
P_TOPRIGHT[0] = s->current_picture.motion_val[0][mot_xy - mot_stride + off[block]][0];
P_TOPRIGHT[1] = s->current_picture.motion_val[0][mot_xy - mot_stride + off[block]][1];
if(P_TOP[1] > (c->ymax<<shift)) P_TOP[1] = (c->ymax<<shift);
if(P_TOPRIGHT[0] < (c->xmin<<shift)) P_TOPRIGHT[0]= (c->xmin<<shift);
if(P_TOPRIGHT[0] > (c->xmax<<shift)) P_TOPRIGHT[0]= (c->xmax<<shift);
if(P_TOPRIGHT[1] > (c->ymax<<shift)) P_TOPRIGHT[1]= (c->ymax<<shift);
if (P_TOP[1] > c->ymax * (1 << shift)) P_TOP[1] = c->ymax * (1 << shift);
if (P_TOPRIGHT[0] < c->xmin * (1 << shift)) P_TOPRIGHT[0] = c->xmin * (1 << shift);
if (P_TOPRIGHT[0] > c->xmax * (1 << shift)) P_TOPRIGHT[0] = c->xmax * (1 << shift);
if (P_TOPRIGHT[1] > c->ymax * (1 << shift)) P_TOPRIGHT[1] = c->ymax * (1 << shift);
P_MEDIAN[0]= mid_pred(P_LEFT[0], P_TOP[0], P_TOPRIGHT[0]);
P_MEDIAN[1]= mid_pred(P_LEFT[1], P_TOP[1], P_TOPRIGHT[1]);
@ -629,8 +629,8 @@ static inline int h263_mv4_search(MpegEncContext *s, int mx, int my, int shift)
continue;
if (i>4 && i<9)
continue;
if(P[i][0] > (c->xmax<<shift)) P[i][0]= (c->xmax<<shift);
if(P[i][1] > (c->ymax<<shift)) P[i][1]= (c->ymax<<shift);
if (P[i][0] > c->xmax * (1 << shift)) P[i][0] = c->xmax * (1 << shift);
if (P[i][1] > c->ymax * (1 << shift)) P[i][1] = c->ymax * (1 <<shift );
}
dmin4 = epzs_motion_search2(s, &mx4, &my4, P, block, block, s->p_mv_table, (1<<16)>>shift, 1);
@ -785,7 +785,7 @@ static int interlaced_search(MpegEncContext *s, int ref_index,
P_TOPRIGHT[0] = mv_table[xy - mot_stride + 1][0];
P_TOPRIGHT[1] = mv_table[xy - mot_stride + 1][1];
if(P_TOP[1] > (c->ymax<<1)) P_TOP[1] = (c->ymax<<1);
if(P_TOPRIGHT[0] < (c->xmin<<1)) P_TOPRIGHT[0]= (c->xmin<<1);
if (P_TOPRIGHT[0] < c->xmin * (1 << 1)) P_TOPRIGHT[0] = c->xmin * (1 << 1);
if(P_TOPRIGHT[0] > (c->xmax<<1)) P_TOPRIGHT[0]= (c->xmax<<1);
if(P_TOPRIGHT[1] > (c->ymax<<1)) P_TOPRIGHT[1]= (c->ymax<<1);
@ -839,7 +839,7 @@ static int interlaced_search(MpegEncContext *s, int ref_index,
dmin_sum += best_dmin;
}
c->ymin<<=1;
c->ymin *= 2;
c->ymax<<=1;
c->stride>>=1;
c->uvstride>>=1;
@ -981,8 +981,8 @@ void ff_estimate_p_frame_motion(MpegEncContext * s,
if(mx || my)
mb_type |= CANDIDATE_MB_TYPE_SKIPPED; //FIXME check difference
}else{
mx <<=shift;
my <<=shift;
mx *= 1 << shift;
my *= 1 << shift;
}
if ((s->avctx->flags & AV_CODEC_FLAG_4MV)
&& !c->skip && varc>50<<8 && vard>10<<8){
@ -1143,7 +1143,7 @@ static int estimate_motion_b(MpegEncContext *s, int mb_x, int mb_y,
P_TOPRIGHT[0] = mv_table[mot_xy - mot_stride + 1][0];
P_TOPRIGHT[1] = mv_table[mot_xy - mot_stride + 1][1];
if (P_TOP[1] > (c->ymax << shift)) P_TOP[1] = (c->ymax << shift);
if (P_TOPRIGHT[0] < (c->xmin << shift)) P_TOPRIGHT[0] = (c->xmin << shift);
if (P_TOPRIGHT[0] < c->xmin * (1 << shift)) P_TOPRIGHT[0] = c->xmin * (1 << shift);
if (P_TOPRIGHT[1] > (c->ymax << shift)) P_TOPRIGHT[1] = (c->ymax << shift);
P_MEDIAN[0] = mid_pred(P_LEFT[0], P_TOP[0], P_TOPRIGHT[0]);
@ -1155,7 +1155,7 @@ static int estimate_motion_b(MpegEncContext *s, int mb_x, int mb_y,
if(mv_table == s->b_forw_mv_table){
mv_scale= (s->pb_time<<16) / (s->pp_time<<shift);
}else{
mv_scale= ((s->pb_time - s->pp_time)<<16) / (s->pp_time<<shift);
mv_scale = ((s->pb_time - s->pp_time) * (1 << 16)) / (s->pp_time<<shift);
}
dmin = ff_epzs_motion_search(s, &mx, &my, P, 0, ref_index, s->p_mv_table, mv_scale, 0, 16);
@ -1255,8 +1255,8 @@ static inline int bidir_refine(MpegEncContext * s, int mb_x, int mb_y)
const int flags= c->sub_flags;
const int qpel= flags&FLAG_QPEL;
const int shift= 1+qpel;
const int xmin= c->xmin<<shift;
const int ymin= c->ymin<<shift;
const int xmin= c->xmin * (1 << shift);
const int ymin= c->ymin * (1 << shift);
const int xmax= c->xmax<<shift;
const int ymax= c->ymax<<shift;
#define HASH(fx,fy,bx,by) ((fx)+17*(fy)+63*(bx)+117*(by))
@ -1454,15 +1454,15 @@ static inline int direct_search(MpegEncContext * s, int mb_x, int mb_y)
c->pred_x=0;
c->pred_y=0;
P_LEFT[0] = av_clip(mv_table[mot_xy - 1][0], xmin<<shift, xmax<<shift);
P_LEFT[1] = av_clip(mv_table[mot_xy - 1][1], ymin<<shift, ymax<<shift);
P_LEFT[0] = av_clip(mv_table[mot_xy - 1][0], xmin * (1 << shift), xmax << shift);
P_LEFT[1] = av_clip(mv_table[mot_xy - 1][1], ymin * (1 << shift), ymax << shift);
/* special case for first line */
if (!s->first_slice_line) { //FIXME maybe allow this over thread boundary as it is clipped
P_TOP[0] = av_clip(mv_table[mot_xy - mot_stride ][0], xmin<<shift, xmax<<shift);
P_TOP[1] = av_clip(mv_table[mot_xy - mot_stride ][1], ymin<<shift, ymax<<shift);
P_TOPRIGHT[0] = av_clip(mv_table[mot_xy - mot_stride + 1 ][0], xmin<<shift, xmax<<shift);
P_TOPRIGHT[1] = av_clip(mv_table[mot_xy - mot_stride + 1 ][1], ymin<<shift, ymax<<shift);
P_TOP[0] = av_clip(mv_table[mot_xy - mot_stride ][0], xmin * (1 << shift), xmax << shift);
P_TOP[1] = av_clip(mv_table[mot_xy - mot_stride ][1], ymin * (1 << shift), ymax << shift);
P_TOPRIGHT[0] = av_clip(mv_table[mot_xy - mot_stride + 1][0], xmin * (1 << shift), xmax << shift);
P_TOPRIGHT[1] = av_clip(mv_table[mot_xy - mot_stride + 1][1], ymin * (1 << shift), ymax << shift);
P_MEDIAN[0]= mid_pred(P_LEFT[0], P_TOP[0], P_TOPRIGHT[0]);
P_MEDIAN[1]= mid_pred(P_LEFT[1], P_TOP[1], P_TOPRIGHT[1]);

17
libavcodec/motion_est_template.c
View File

@ -82,7 +82,7 @@ static int hpel_motion_search(MpegEncContext * s,
if (mx > xmin && mx < xmax &&
my > ymin && my < ymax) {
int d= dmin;
const int index= (my<<ME_MAP_SHIFT) + mx;
const int index = my * (1 << ME_MAP_SHIFT) + mx;
const int t= score_map[(index-(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)]
+ (mv_penalty[bx - pred_x] + mv_penalty[by-2 - pred_y])*c->penalty_factor;
const int l= score_map[(index- 1 )&(ME_MAP_SIZE-1)]
@ -95,13 +95,13 @@ static int hpel_motion_search(MpegEncContext * s,
#if defined(ASSERT_LEVEL) && ASSERT_LEVEL > 1
unsigned key;
unsigned map_generation= c->map_generation;
key= ((my-1)<<ME_MAP_MV_BITS) + (mx) + map_generation;
key = (my - 1) * (1 << ME_MAP_MV_BITS) + (mx) + map_generation;
av_assert2(c->map[(index-(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)] == key);
key= ((my+1)<<ME_MAP_MV_BITS) + (mx) + map_generation;
key = (my + 1) * (1 << ME_MAP_MV_BITS) + (mx) + map_generation;
av_assert2(c->map[(index+(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)] == key);
key= ((my)<<ME_MAP_MV_BITS) + (mx+1) + map_generation;
key = (my) * (1 << ME_MAP_MV_BITS) + (mx + 1) + map_generation;
av_assert2(c->map[(index+1)&(ME_MAP_SIZE-1)] == key);
key= ((my)<<ME_MAP_MV_BITS) + (mx-1) + map_generation;
key = (my) * (1 << ME_MAP_MV_BITS) + (mx - 1) + map_generation;
av_assert2(c->map[(index-1)&(ME_MAP_SIZE-1)] == key);
#endif
if(t<=b){
@ -246,7 +246,7 @@ static int qpel_motion_search(MpegEncContext * s,
int bx=4*mx, by=4*my;
int d= dmin;
int i, nx, ny;
const int index= (my<<ME_MAP_SHIFT) + mx;
const int index = my * (1 << ME_MAP_SHIFT) + mx;
const int t= score_map[(index-(1<<ME_MAP_SHIFT) )&(ME_MAP_SIZE-1)];
const int l= score_map[(index- 1 )&(ME_MAP_SIZE-1)];
const int r= score_map[(index+ 1 )&(ME_MAP_SIZE-1)];
@ -299,7 +299,8 @@ static int qpel_motion_search(MpegEncContext * s,
const int cy2= b + t - 2*c;
int cxy;
if(map[(index-(1<<ME_MAP_SHIFT)-1)&(ME_MAP_SIZE-1)] == ((my-1)<<ME_MAP_MV_BITS) + (mx-1) + map_generation){
if (map[(index - (1 << ME_MAP_SHIFT) - 1) & (ME_MAP_SIZE - 1)] ==
(my - 1) * (1 << ME_MAP_MV_BITS) + (mx - 1) + map_generation) {
tl= score_map[(index-(1<<ME_MAP_SHIFT)-1)&(ME_MAP_SIZE-1)];
}else{
tl= cmp(s, mx-1, my-1, 0, 0, size, h, ref_index, src_index, cmpf, chroma_cmpf, flags);//FIXME wrong if chroma me is different
@ -328,7 +329,7 @@ static int qpel_motion_search(MpegEncContext * s,
for(i=0; i<8; i++){
if(score < best[i]){
memmove(&best[i+1], &best[i], sizeof(int)*(7-i));
memmove(&best_pos[i+1][0], &best_pos[i][0], sizeof(int)*2*(7-i));
memmove(best_pos[i + 1], best_pos[i], sizeof(best_pos[0]) * (7 - i));
best[i]= score;
best_pos[i][0]= nx + 4*mx;
best_pos[i][1]= ny + 4*my;