har0ke/FFmpeg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

H.264: fix overreads of qscale_table

Browse Source 
filter_mb_fast assumed that qscale_table was padded like many of the other tables.
This commit is contained in:
Jason Garrett-Glaser 2011-07-04 06:05:34 -07:00
parent e589e4b82d
commit 5029a40633
2 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
libavcodec/mpegvideo.c
View File

@ -285,9 +285,10 @@ int ff_alloc_picture(MpegEncContext *s, Picture *pic, int shared){
} }
FF_ALLOCZ_OR_GOTO(s->avctx, pic->mbskip_table , mb_array_size * sizeof(uint8_t)+2, fail) //the +2 is for the slice end check FF_ALLOCZ_OR_GOTO(s->avctx, pic->mbskip_table , mb_array_size * sizeof(uint8_t)+2, fail) //the +2 is for the slice end check
FF_ALLOCZ_OR_GOTO(s->avctx, pic->qscale_table , mb_array_size * sizeof(uint8_t) , fail) FF_ALLOCZ_OR_GOTO(s->avctx, pic->qscale_table_base , (big_mb_num + s->mb_stride) * sizeof(uint8_t) , fail)
FF_ALLOCZ_OR_GOTO(s->avctx, pic->mb_type_base , (big_mb_num + s->mb_stride) * sizeof(uint32_t), fail) FF_ALLOCZ_OR_GOTO(s->avctx, pic->mb_type_base , (big_mb_num + s->mb_stride) * sizeof(uint32_t), fail)
pic->mb_type= pic->mb_type_base + 2*s->mb_stride+1; pic->mb_type= pic->mb_type_base + 2*s->mb_stride+1;
pic->qscale_table = pic->qscale_table_base + 2*s->mb_stride + 1;
if(s->out_format == FMT_H264){ if(s->out_format == FMT_H264){
for(i=0; i<2; i++){ for(i=0; i<2; i++){
FF_ALLOCZ_OR_GOTO(s->avctx, pic->motion_val_base[i], 2 * (b4_array_size+4) * sizeof(int16_t), fail) FF_ALLOCZ_OR_GOTO(s->avctx, pic->motion_val_base[i], 2 * (b4_array_size+4) * sizeof(int16_t), fail)
@ -339,7 +340,7 @@ static void free_picture(MpegEncContext *s, Picture *pic){
av_freep(&pic->mc_mb_var); av_freep(&pic->mc_mb_var);
av_freep(&pic->mb_mean); av_freep(&pic->mb_mean);
av_freep(&pic->mbskip_table); av_freep(&pic->mbskip_table);
av_freep(&pic->qscale_table); av_freep(&pic->qscale_table_base);
av_freep(&pic->mb_type_base); av_freep(&pic->mb_type_base);
av_freep(&pic->dct_coeff); av_freep(&pic->dct_coeff);
av_freep(&pic->pan_scan); av_freep(&pic->pan_scan);

1
libavcodec/mpegvideo.h
View File

@ -88,6 +88,7 @@ typedef struct Picture{
* halfpel luma planes. * halfpel luma planes.
*/ */
uint8_t *interpolated[3]; uint8_t *interpolated[3];
int8_t *qscale_table_base;
int16_t (*motion_val_base[2])[2]; int16_t (*motion_val_base[2])[2];
uint32_t *mb_type_base; uint32_t *mb_type_base;
#define MB_TYPE_INTRA MB_TYPE_INTRA4x4 //default mb_type if there is just one type #define MB_TYPE_INTRA MB_TYPE_INTRA4x4 //default mb_type if there is just one type