ffmpeg: Use hardware config metadata with encoders

This can support encoders which want frames and/or device contexts.  For
the device case, it currently picks the first initialised device of the
desired type to give to the encoder - a new option would be needed if it
were necessary to choose between multiple devices of the same type.

fftools/ffmpeg.c

@@ -3476,21 +3476,14 @@ static int init_output_stream(OutputStream *ost, char *error, int error_len)
             !av_dict_get(ost->encoder_opts, "ab", NULL, 0))
             av_dict_set(&ost->encoder_opts, "b", "128000", 0);
 
-        if (ost->filter && av_buffersink_get_hw_frames_ctx(ost->filter->filter) &&
+        ret = hw_device_setup_for_encode(ost);
-            ((AVHWFramesContext*)av_buffersink_get_hw_frames_ctx(ost->filter->filter)->data)->format ==
+        if (ret < 0) {
-            av_buffersink_get_format(ost->filter->filter)) {
+            snprintf(error, error_len, "Device setup failed for "
-            ost->enc_ctx->hw_frames_ctx = av_buffer_ref(av_buffersink_get_hw_frames_ctx(ost->filter->filter));
+                     "encoder on output stream #%d:%d : %s",
-            if (!ost->enc_ctx->hw_frames_ctx)
-                return AVERROR(ENOMEM);
-        } else {
-            ret = hw_device_setup_for_encode(ost);
-            if (ret < 0) {
-                snprintf(error, error_len, "Device setup failed for "
-                         "encoder on output stream #%d:%d : %s",
                      ost->file_index, ost->index, av_err2str(ret));
-                return ret;
+            return ret;
-            }
         }
+
         if (ist && ist->dec->type == AVMEDIA_TYPE_SUBTITLE && ost->enc->type == AVMEDIA_TYPE_SUBTITLE) {
             int input_props = 0, output_props = 0;
             AVCodecDescriptor const *input_descriptor =

fftools/ffmpeg_hw.c

@@ -19,6 +19,7 @@
 #include <string.h>
 
 #include "libavutil/avstring.h"
+#include "libavfilter/buffersink.h"
 
 #include "ffmpeg.h"
 
@@ -281,7 +282,10 @@ void hw_device_free_all(void)
     nb_hw_devices = 0;
 }
 
-static HWDevice *hw_device_match_by_codec(const AVCodec *codec)
+static HWDevice *hw_device_match_by_codec(const AVCodec *codec,
+                                          enum AVPixelFormat format,
+                                          int possible_methods,
+                                          int *matched_methods)
 {
     const AVCodecHWConfig *config;
     HWDevice *dev;
@@ -290,11 +294,18 @@ static HWDevice *hw_device_match_by_codec(const AVCodec *codec)
         config = avcodec_get_hw_config(codec, i);
         if (!config)
             return NULL;
-        if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX))
+        if (format != AV_PIX_FMT_NONE &&
+            config->pix_fmt != AV_PIX_FMT_NONE &&
+            config->pix_fmt != format)
+            continue;
+        if (!(config->methods & possible_methods))
             continue;
         dev = hw_device_get_by_type(config->device_type);
-        if (dev)
+        if (dev) {
+            if (matched_methods)
+                *matched_methods = config->methods & possible_methods;
             return dev;
+        }
     }
 }
 
@@ -340,7 +351,9 @@ int hw_device_setup_for_decode(InputStream *ist)
             if (!dev)
                 err = hw_device_init_from_type(type, NULL, &dev);
         } else {
-            dev = hw_device_match_by_codec(ist->dec);
+            dev = hw_device_match_by_codec(ist->dec, AV_PIX_FMT_NONE,
+                                           AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX,
+                                           NULL);
             if (!dev) {
                 // No device for this codec, but not using generic hwaccel
                 // and therefore may well not need one - ignore.
@@ -417,12 +430,31 @@ int hw_device_setup_for_decode(InputStream *ist)
 int hw_device_setup_for_encode(OutputStream *ost)
 {
     HWDevice *dev;
+    AVBufferRef *frames_ref;
+    int methods = AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX;
+    int matched_methods;
 
-    dev = hw_device_match_by_codec(ost->enc);
+    if (ost->filter) {
+        frames_ref = av_buffersink_get_hw_frames_ctx(ost->filter->filter);
+        if (frames_ref &&
+            ((AVHWFramesContext*)frames_ref->data)->format ==
+            ost->enc_ctx->pix_fmt)
+            methods |= AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX;
+    }
+
+    dev = hw_device_match_by_codec(ost->enc, ost->enc_ctx->pix_fmt,
+                                   methods, &matched_methods);
     if (dev) {
-        ost->enc_ctx->hw_device_ctx = av_buffer_ref(dev->device_ref);
+        if (matched_methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) {
-        if (!ost->enc_ctx->hw_device_ctx)
+            ost->enc_ctx->hw_device_ctx = av_buffer_ref(dev->device_ref);
-            return AVERROR(ENOMEM);
+            if (!ost->enc_ctx->hw_device_ctx)
+                return AVERROR(ENOMEM);
+        }
+        if (matched_methods & AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX) {
+            ost->enc_ctx->hw_frames_ctx = av_buffer_ref(frames_ref);
+            if (!ost->enc_ctx->hw_frames_ctx)
+                return AVERROR(ENOMEM);
+        }
         return 0;
     } else {
         // No device required, or no device available.