avfilter/formats: Make ff_formats_pixdesc_filter return AVFilterFormats*

Up until now, it has returned the AVFilterFormats list via
an AVFilterFormats** parameter; the actual return value was an int
that was always AVERROR(ENOMEM) on error. The AVFilterFormats**
argument was a pure output parameter which was only documented
by naming the parameter rfmts. Yet nevertheless all callers
initialized the underlying AVFilterFormats* to NULL.

This commit changes this to return a pointer to AVFilterFormats
directly. This is more in line with the API in general, as it
allows to avoid checks for intermediate values.

Reviewed-by: Nicolas George <george@nsup.org>
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>

libavfilter/formats.c

@@ -452,7 +452,7 @@ AVFilterFormats *ff_all_formats(enum AVMediaType type)
     return ret;
 }
 
-int ff_formats_pixdesc_filter(AVFilterFormats **rfmts, unsigned want, unsigned rej)
+AVFilterFormats *ff_formats_pixdesc_filter(unsigned want, unsigned rej)
 {
     unsigned nb_formats, fmt, flags;
     AVFilterFormats *formats = NULL;
@@ -476,18 +476,17 @@ int ff_formats_pixdesc_filter(AVFilterFormats **rfmts, unsigned want, unsigned r
         }
         if (formats) {
             av_assert0(formats->nb_formats == nb_formats);
-            *rfmts = formats;
+            return formats;
-            return 0;
         }
         formats = av_mallocz(sizeof(*formats));
         if (!formats)
-            return AVERROR(ENOMEM);
+            return NULL;
         formats->nb_formats = nb_formats;
         if (nb_formats) {
             formats->formats = av_malloc_array(nb_formats, sizeof(*formats->formats));
             if (!formats->formats) {
                 av_freep(&formats);
-                return AVERROR(ENOMEM);
+                return NULL;
             }
         }
     }

libavfilter/formats.h

@@ -230,7 +230,7 @@ AVFilterFormats *ff_all_formats(enum AVMediaType type);
  * properties
  */
 av_warn_unused_result
-int ff_formats_pixdesc_filter(AVFilterFormats **rfmts, unsigned want, unsigned rej);
+AVFilterFormats *ff_formats_pixdesc_filter(unsigned want, unsigned rej);
 
 //* format is software, non-planar with sub-sampling
 #define FF_PIX_FMT_FLAG_SW_FLAT_SUB (1 << 24)

libavfilter/vf_copy.c

@@ -29,14 +29,7 @@
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, AV_PIX_FMT_FLAG_HWACCEL));
-    int ret;
-
-    ret = ff_formats_pixdesc_filter(&formats, 0,
-                                    AV_PIX_FMT_FLAG_HWACCEL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
 }
 
 static int filter_frame(AVFilterLink *inlink, AVFrame *in)

libavfilter/vf_crop.c

@@ -93,13 +93,9 @@ typedef struct CropContext {
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
+    int reject_flags = AV_PIX_FMT_FLAG_BITSTREAM | FF_PIX_FMT_FLAG_SW_FLAT_SUB;
-    int ret;
 
-    ret = ff_formats_pixdesc_filter(&formats, 0, AV_PIX_FMT_FLAG_BITSTREAM | FF_PIX_FMT_FLAG_SW_FLAT_SUB);
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
 }
 
 static av_cold void uninit(AVFilterContext *ctx)

libavfilter/vf_detelecine.c

@@ -124,16 +124,11 @@ static av_cold int init(AVFilterContext *ctx)
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
+    int reject_flags = AV_PIX_FMT_FLAG_BITSTREAM |
-    int ret;
+                       AV_PIX_FMT_FLAG_PAL       |
+                       AV_PIX_FMT_FLAG_HWACCEL;
 
-    ret = ff_formats_pixdesc_filter(&formats, 0,
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
-                                    AV_PIX_FMT_FLAG_BITSTREAM |
-                                    AV_PIX_FMT_FLAG_PAL |
-                                    AV_PIX_FMT_FLAG_HWACCEL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
 }
 
 static int config_input(AVFilterLink *inlink)

libavfilter/vf_fieldhint.c

@@ -77,16 +77,11 @@ static av_cold int init(AVFilterContext *ctx)
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
+    int reject_flags = AV_PIX_FMT_FLAG_BITSTREAM |
-    int ret;
+                       AV_PIX_FMT_FLAG_HWACCEL   |
+                       AV_PIX_FMT_FLAG_PAL;
 
-    ret = ff_formats_pixdesc_filter(&formats, 0,
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
-                                    AV_PIX_FMT_FLAG_HWACCEL |
-                                    AV_PIX_FMT_FLAG_BITSTREAM |
-                                    AV_PIX_FMT_FLAG_PAL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
 }
 
 static int config_input(AVFilterLink *inlink)

libavfilter/vf_hwdownload.c

@@ -37,13 +37,12 @@ typedef struct HWDownloadContext {
 
 static int hwdownload_query_formats(AVFilterContext *avctx)
 {
-    AVFilterFormats *fmts;
     int err;
 
-    if ((err = ff_formats_pixdesc_filter(&fmts, AV_PIX_FMT_FLAG_HWACCEL, 0)) ||
+    if ((err = ff_formats_ref(ff_formats_pixdesc_filter(AV_PIX_FMT_FLAG_HWACCEL, 0),
-        (err = ff_formats_ref(fmts, &avctx->inputs[0]->outcfg.formats))      ||
+                              &avctx->inputs[0]->outcfg.formats))  ||
-        (err = ff_formats_pixdesc_filter(&fmts, 0, AV_PIX_FMT_FLAG_HWACCEL)) ||
+        (err = ff_formats_ref(ff_formats_pixdesc_filter(0, AV_PIX_FMT_FLAG_HWACCEL),
-        (err = ff_formats_ref(fmts, &avctx->outputs[0]->incfg.formats)))
+                              &avctx->outputs[0]->incfg.formats)))
         return err;
 
     return 0;

libavfilter/vf_il.c

@@ -83,15 +83,9 @@ AVFILTER_DEFINE_CLASS(il);
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
+    int reject_flags = AV_PIX_FMT_FLAG_PAL | AV_PIX_FMT_FLAG_HWACCEL;
-    int ret;
 
-    ret = ff_formats_pixdesc_filter(&formats, 0,
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
-                                    AV_PIX_FMT_FLAG_PAL |
-                                    AV_PIX_FMT_FLAG_HWACCEL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
 }
 
 static int config_input(AVFilterLink *inlink)

libavfilter/vf_mix.c

@@ -55,16 +55,11 @@ typedef struct MixContext {
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
+    int reject_flags = AV_PIX_FMT_FLAG_BITSTREAM |
-    int ret;
+                       AV_PIX_FMT_FLAG_HWACCEL   |
+                       AV_PIX_FMT_FLAG_PAL;
 
-    ret = ff_formats_pixdesc_filter(&formats, 0,
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
-                                    AV_PIX_FMT_FLAG_BITSTREAM |
-                                    AV_PIX_FMT_FLAG_PAL |
-                                    AV_PIX_FMT_FLAG_HWACCEL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
 }
 
 static int parse_weights(AVFilterContext *ctx)

libavfilter/vf_stack.c

@@ -60,21 +60,16 @@ typedef struct StackContext {
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
     StackContext *s = ctx->priv;
-    int ret;
+    int reject_flags = AV_PIX_FMT_FLAG_BITSTREAM |
+                       AV_PIX_FMT_FLAG_HWACCEL   |
+                       AV_PIX_FMT_FLAG_PAL;
 
     if (s->fillcolor_enable) {
         return ff_set_common_formats(ctx, ff_draw_supported_pixel_formats(0));
     }
 
-    ret = ff_formats_pixdesc_filter(&formats, 0,
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
-                                    AV_PIX_FMT_FLAG_HWACCEL |
-                                    AV_PIX_FMT_FLAG_BITSTREAM |
-                                    AV_PIX_FMT_FLAG_PAL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
 }
 
 static av_cold int init(AVFilterContext *ctx)

libavfilter/vf_swaprect.c

@@ -57,16 +57,11 @@ AVFILTER_DEFINE_CLASS(swaprect);
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *pix_fmts = NULL;
+    int reject_flags = AV_PIX_FMT_FLAG_PAL     |
-    int ret;
+                       AV_PIX_FMT_FLAG_HWACCEL |
+                       AV_PIX_FMT_FLAG_BITSTREAM;
 
-    ret = ff_formats_pixdesc_filter(&pix_fmts, 0, AV_PIX_FMT_FLAG_PAL     |
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
-                                                  AV_PIX_FMT_FLAG_HWACCEL |
-                                                  AV_PIX_FMT_FLAG_BITSTREAM);
-    if (ret < 0)
-        return ret;
-
-    return ff_set_common_formats(ctx, pix_fmts);
 }
 
 static const char *const var_names[] = {   "w",   "h",   "a",   "n",   "t",   "pos",   "sar",   "dar",        NULL };

libavfilter/vf_telecine.c

@@ -101,16 +101,11 @@ static av_cold int init(AVFilterContext *ctx)
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
+    int reject_flags = AV_PIX_FMT_FLAG_BITSTREAM |
-    int ret;
+                       AV_PIX_FMT_FLAG_HWACCEL   |
+                       AV_PIX_FMT_FLAG_PAL;
 
-    ret = ff_formats_pixdesc_filter(&formats, 0,
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
-                                    AV_PIX_FMT_FLAG_BITSTREAM |
-                                    AV_PIX_FMT_FLAG_PAL |
-                                    AV_PIX_FMT_FLAG_HWACCEL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
 }
 
 static int config_input(AVFilterLink *inlink)

libavfilter/vf_untile.c

@@ -62,16 +62,11 @@ static av_cold int init(AVFilterContext *ctx)
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
+    int reject_flags = AV_PIX_FMT_FLAG_HWACCEL   |
-    int ret;
+                       AV_PIX_FMT_FLAG_BITSTREAM |
+                       FF_PIX_FMT_FLAG_SW_FLAT_SUB;
 
-    ret = ff_formats_pixdesc_filter(&formats, 0,
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
-                                    AV_PIX_FMT_FLAG_HWACCEL |
-                                    AV_PIX_FMT_FLAG_BITSTREAM |
-                                    FF_PIX_FMT_FLAG_SW_FLAT_SUB);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
 }
 
 static int config_output(AVFilterLink *outlink)

libavfilter/vf_weave.c

@@ -51,15 +51,9 @@ AVFILTER_DEFINE_CLASS_EXT(weave, "(double)weave", weave_options);
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
+    int reject_flags = AV_PIX_FMT_FLAG_PAL | AV_PIX_FMT_FLAG_HWACCEL;
-    int ret;
 
-    ret = ff_formats_pixdesc_filter(&formats, 0,
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
-                                    AV_PIX_FMT_FLAG_PAL |
-                                    AV_PIX_FMT_FLAG_HWACCEL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
 }
 
 static int config_props_output(AVFilterLink *outlink)