From b4b2f88cabf99c5928b7593cfe4f12d1c9b59f4f Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
Date: Fri, 26 Mar 2021 18:18:05 +0100
Subject: [PATCH] avcodec/motion_est: Fix invalid left shift of negative
 numbers

Affected many FATE-tests.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
(cherry picked from commit 3ef65fd4d1d0cdfa0b60351c719bef93d3664ea2)
---
 libavcodec/motion_est.c          | 46 ++++++++++++++++----------------
 libavcodec/motion_est_template.c | 17 ++++++------
 2 files changed, 32 insertions(+), 31 deletions(-)

diff --git a/libavcodec/motion_est.c b/libavcodec/motion_est.c
index 5a339fc1c7..5b0958733c 100644
--- a/libavcodec/motion_est.c
+++ b/libavcodec/motion_est.c
@@ -109,8 +109,8 @@ static av_always_inline int cmp_direct_inline(MpegEncContext *s, const int x, co
                       me_cmp_func cmp_func, me_cmp_func chroma_cmp_func, int qpel){
     MotionEstContext * const c= &s->me;
     const int stride= c->stride;
-    const int hx= subx + (x<<(1+qpel));
-    const int hy= suby + (y<<(1+qpel));
+    const int hx = subx + x * (1 << (1 + qpel));
+    const int hy = suby + y * (1 << (1 + qpel));
     uint8_t * const * const ref= c->ref[ref_index];
     uint8_t * const * const src= c->src[src_index];
     int d;
@@ -599,7 +599,7 @@ static inline int h263_mv4_search(MpegEncContext *s, int mx, int my, int shift)
         P_LEFT[0] = s->current_picture.motion_val[0][mot_xy - 1][0];
         P_LEFT[1] = s->current_picture.motion_val[0][mot_xy - 1][1];
 
-        if(P_LEFT[0]       > (c->xmax<<shift)) P_LEFT[0]       = (c->xmax<<shift);
+        if (P_LEFT[0] > c->xmax * (1 << shift)) P_LEFT[0] = c->xmax * (1 << shift);
 
         /* special case for first line */
         if (s->first_slice_line && block<2) {
@@ -610,10 +610,10 @@ static inline int h263_mv4_search(MpegEncContext *s, int mx, int my, int shift)
             P_TOP[1]      = s->current_picture.motion_val[0][mot_xy - mot_stride             ][1];
             P_TOPRIGHT[0] = s->current_picture.motion_val[0][mot_xy - mot_stride + off[block]][0];
             P_TOPRIGHT[1] = s->current_picture.motion_val[0][mot_xy - mot_stride + off[block]][1];
-            if(P_TOP[1]      > (c->ymax<<shift)) P_TOP[1]     = (c->ymax<<shift);
-            if(P_TOPRIGHT[0] < (c->xmin<<shift)) P_TOPRIGHT[0]= (c->xmin<<shift);
-            if(P_TOPRIGHT[0] > (c->xmax<<shift)) P_TOPRIGHT[0]= (c->xmax<<shift);
-            if(P_TOPRIGHT[1] > (c->ymax<<shift)) P_TOPRIGHT[1]= (c->ymax<<shift);
+            if (P_TOP[1]      > c->ymax * (1 << shift)) P_TOP[1]      = c->ymax * (1 << shift);
+            if (P_TOPRIGHT[0] < c->xmin * (1 << shift)) P_TOPRIGHT[0] = c->xmin * (1 << shift);
+            if (P_TOPRIGHT[0] > c->xmax * (1 << shift)) P_TOPRIGHT[0] = c->xmax * (1 << shift);
+            if (P_TOPRIGHT[1] > c->ymax * (1 << shift)) P_TOPRIGHT[1] = c->ymax * (1 << shift);
 
             P_MEDIAN[0]= mid_pred(P_LEFT[0], P_TOP[0], P_TOPRIGHT[0]);
             P_MEDIAN[1]= mid_pred(P_LEFT[1], P_TOP[1], P_TOPRIGHT[1]);
@@ -629,8 +629,8 @@ static inline int h263_mv4_search(MpegEncContext *s, int mx, int my, int shift)
                     continue;
                 if (i>4 && i<9)
                     continue;
-                if(P[i][0] > (c->xmax<<shift)) P[i][0]= (c->xmax<<shift);
-                if(P[i][1] > (c->ymax<<shift)) P[i][1]= (c->ymax<<shift);
+                if (P[i][0] > c->xmax * (1 << shift)) P[i][0] = c->xmax * (1 << shift);
+                if (P[i][1] > c->ymax * (1 << shift)) P[i][1] = c->ymax * (1 <<shift );
             }
 
         dmin4 = epzs_motion_search2(s, &mx4, &my4, P, block, block, s->p_mv_table, (1<<16)>>shift, 1);
@@ -785,7 +785,7 @@ static int interlaced_search(MpegEncContext *s, int ref_index,
                 P_TOPRIGHT[0] = mv_table[xy - mot_stride + 1][0];
                 P_TOPRIGHT[1] = mv_table[xy - mot_stride + 1][1];
                 if(P_TOP[1]      > (c->ymax<<1)) P_TOP[1]     = (c->ymax<<1);
-                if(P_TOPRIGHT[0] < (c->xmin<<1)) P_TOPRIGHT[0]= (c->xmin<<1);
+                if (P_TOPRIGHT[0] < c->xmin * (1 << 1)) P_TOPRIGHT[0] = c->xmin * (1 << 1);
                 if(P_TOPRIGHT[0] > (c->xmax<<1)) P_TOPRIGHT[0]= (c->xmax<<1);
                 if(P_TOPRIGHT[1] > (c->ymax<<1)) P_TOPRIGHT[1]= (c->ymax<<1);
 
@@ -839,7 +839,7 @@ static int interlaced_search(MpegEncContext *s, int ref_index,
         dmin_sum += best_dmin;
     }
 
-    c->ymin<<=1;
+    c->ymin *= 2;
     c->ymax<<=1;
     c->stride>>=1;
     c->uvstride>>=1;
@@ -981,8 +981,8 @@ void ff_estimate_p_frame_motion(MpegEncContext * s,
                 if(mx || my)
                     mb_type |= CANDIDATE_MB_TYPE_SKIPPED; //FIXME check difference
         }else{
-            mx <<=shift;
-            my <<=shift;
+            mx *= 1 << shift;
+            my *= 1 << shift;
         }
         if ((s->avctx->flags & AV_CODEC_FLAG_4MV)
            && !c->skip && varc>50<<8 && vard>10<<8){
@@ -1143,7 +1143,7 @@ static int estimate_motion_b(MpegEncContext *s, int mb_x, int mb_y,
             P_TOPRIGHT[0] = mv_table[mot_xy - mot_stride + 1][0];
             P_TOPRIGHT[1] = mv_table[mot_xy - mot_stride + 1][1];
             if (P_TOP[1] > (c->ymax << shift)) P_TOP[1] = (c->ymax << shift);
-            if (P_TOPRIGHT[0] < (c->xmin << shift)) P_TOPRIGHT[0] = (c->xmin << shift);
+            if (P_TOPRIGHT[0] < c->xmin * (1 << shift)) P_TOPRIGHT[0] = c->xmin * (1 << shift);
             if (P_TOPRIGHT[1] > (c->ymax << shift)) P_TOPRIGHT[1] = (c->ymax << shift);
 
             P_MEDIAN[0] = mid_pred(P_LEFT[0], P_TOP[0], P_TOPRIGHT[0]);
@@ -1155,7 +1155,7 @@ static int estimate_motion_b(MpegEncContext *s, int mb_x, int mb_y,
         if(mv_table == s->b_forw_mv_table){
             mv_scale= (s->pb_time<<16) / (s->pp_time<<shift);
         }else{
-            mv_scale= ((s->pb_time - s->pp_time)<<16) / (s->pp_time<<shift);
+            mv_scale = ((s->pb_time - s->pp_time) * (1 << 16)) / (s->pp_time<<shift);
         }
 
         dmin = ff_epzs_motion_search(s, &mx, &my, P, 0, ref_index, s->p_mv_table, mv_scale, 0, 16);
@@ -1255,8 +1255,8 @@ static inline int bidir_refine(MpegEncContext * s, int mb_x, int mb_y)
     const int flags= c->sub_flags;
     const int qpel= flags&FLAG_QPEL;
     const int shift= 1+qpel;
-    const int xmin= c->xmin<<shift;
-    const int ymin= c->ymin<<shift;
+    const int xmin= c->xmin * (1 << shift);
+    const int ymin= c->ymin * (1 << shift);
     const int xmax= c->xmax<<shift;
     const int ymax= c->ymax<<shift;
 #define HASH(fx,fy,bx,by) ((fx)+17*(fy)+63*(bx)+117*(by))
@@ -1454,15 +1454,15 @@ static inline int direct_search(MpegEncContext * s, int mb_x, int mb_y)
     c->pred_x=0;
     c->pred_y=0;
 
-    P_LEFT[0]        = av_clip(mv_table[mot_xy - 1][0], xmin<<shift, xmax<<shift);
-    P_LEFT[1]        = av_clip(mv_table[mot_xy - 1][1], ymin<<shift, ymax<<shift);
+    P_LEFT[0] = av_clip(mv_table[mot_xy - 1][0], xmin * (1 << shift), xmax << shift);
+    P_LEFT[1] = av_clip(mv_table[mot_xy - 1][1], ymin * (1 << shift), ymax << shift);
 
     /* special case for first line */
     if (!s->first_slice_line) { //FIXME maybe allow this over thread boundary as it is clipped
-        P_TOP[0]      = av_clip(mv_table[mot_xy - mot_stride             ][0], xmin<<shift, xmax<<shift);
-        P_TOP[1]      = av_clip(mv_table[mot_xy - mot_stride             ][1], ymin<<shift, ymax<<shift);
-        P_TOPRIGHT[0] = av_clip(mv_table[mot_xy - mot_stride + 1         ][0], xmin<<shift, xmax<<shift);
-        P_TOPRIGHT[1] = av_clip(mv_table[mot_xy - mot_stride + 1         ][1], ymin<<shift, ymax<<shift);
+        P_TOP[0]      = av_clip(mv_table[mot_xy - mot_stride    ][0], xmin * (1 << shift), xmax << shift);
+        P_TOP[1]      = av_clip(mv_table[mot_xy - mot_stride    ][1], ymin * (1 << shift), ymax << shift);
+        P_TOPRIGHT[0] = av_clip(mv_table[mot_xy - mot_stride + 1][0], xmin * (1 << shift), xmax << shift);
+        P_TOPRIGHT[1] = av_clip(mv_table[mot_xy - mot_stride + 1][1], ymin * (1 << shift), ymax << shift);
 
         P_MEDIAN[0]= mid_pred(P_LEFT[0], P_TOP[0], P_TOPRIGHT[0]);
         P_MEDIAN[1]= mid_pred(P_LEFT[1], P_TOP[1], P_TOPRIGHT[1]);
diff --git a/libavcodec/motion_est_template.c b/libavcodec/motion_est_template.c
index d3b2f97744..6ab0ea13dc 100644
--- a/libavcodec/motion_est_template.c
+++ b/libavcodec/motion_est_template.c
@@ -82,7 +82,7 @@ static int hpel_motion_search(MpegEncContext * s,
     if (mx > xmin && mx < xmax &&
         my > ymin && my < ymax) {
         int d= dmin;
-        const int index= (my<<ME_MAP_SHIFT) + mx;
+        const int index = my * (1 << ME_MAP_SHIFT) + mx;
         const int t= score_map[(index-(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)]
                      + (mv_penalty[bx   - pred_x] + mv_penalty[by-2 - pred_y])*c->penalty_factor;
         const int l= score_map[(index- 1               )&(ME_MAP_SIZE-1)]
@@ -95,13 +95,13 @@ static int hpel_motion_search(MpegEncContext * s,
 #if defined(ASSERT_LEVEL) && ASSERT_LEVEL > 1
         unsigned key;
         unsigned map_generation= c->map_generation;
-        key= ((my-1)<<ME_MAP_MV_BITS) + (mx) + map_generation;
+        key = (my - 1) * (1 << ME_MAP_MV_BITS) + (mx) + map_generation;
         av_assert2(c->map[(index-(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)] == key);
-        key= ((my+1)<<ME_MAP_MV_BITS) + (mx) + map_generation;
+        key = (my + 1) * (1 << ME_MAP_MV_BITS) + (mx) + map_generation;
         av_assert2(c->map[(index+(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)] == key);
-        key= ((my)<<ME_MAP_MV_BITS) + (mx+1) + map_generation;
+        key = (my) * (1 << ME_MAP_MV_BITS) + (mx + 1) + map_generation;
         av_assert2(c->map[(index+1)&(ME_MAP_SIZE-1)] == key);
-        key= ((my)<<ME_MAP_MV_BITS) + (mx-1) + map_generation;
+        key = (my) * (1 << ME_MAP_MV_BITS) + (mx - 1) + map_generation;
         av_assert2(c->map[(index-1)&(ME_MAP_SIZE-1)] == key);
 #endif
         if(t<=b){
@@ -246,7 +246,7 @@ static int qpel_motion_search(MpegEncContext * s,
         int bx=4*mx, by=4*my;
         int d= dmin;
         int i, nx, ny;
-        const int index= (my<<ME_MAP_SHIFT) + mx;
+        const int index = my * (1 << ME_MAP_SHIFT) + mx;
         const int t= score_map[(index-(1<<ME_MAP_SHIFT)  )&(ME_MAP_SIZE-1)];
         const int l= score_map[(index- 1                 )&(ME_MAP_SIZE-1)];
         const int r= score_map[(index+ 1                 )&(ME_MAP_SIZE-1)];
@@ -299,7 +299,8 @@ static int qpel_motion_search(MpegEncContext * s,
             const int cy2= b + t - 2*c;
             int cxy;
 
-            if(map[(index-(1<<ME_MAP_SHIFT)-1)&(ME_MAP_SIZE-1)] == ((my-1)<<ME_MAP_MV_BITS) + (mx-1) + map_generation){
+            if (map[(index - (1 << ME_MAP_SHIFT) - 1) & (ME_MAP_SIZE - 1)] ==
+                (my - 1) * (1 << ME_MAP_MV_BITS) + (mx - 1) + map_generation) {
                 tl= score_map[(index-(1<<ME_MAP_SHIFT)-1)&(ME_MAP_SIZE-1)];
             }else{
                 tl= cmp(s, mx-1, my-1, 0, 0, size, h, ref_index, src_index, cmpf, chroma_cmpf, flags);//FIXME wrong if chroma me is different
@@ -328,7 +329,7 @@ static int qpel_motion_search(MpegEncContext * s,
                     for(i=0; i<8; i++){
                         if(score < best[i]){
                             memmove(&best[i+1], &best[i], sizeof(int)*(7-i));
-                            memmove(&best_pos[i+1][0], &best_pos[i][0], sizeof(int)*2*(7-i));
+                            memmove(best_pos[i + 1], best_pos[i], sizeof(best_pos[0]) * (7 - i));
                             best[i]= score;
                             best_pos[i][0]= nx + 4*mx;
                             best_pos[i][1]= ny + 4*my;