lavc: add a codec flag for propagating opaque from frames to packets

This is intended to be a more convenient replacement for
reordered_opaque.

Add support for it in the two encoders that offer
AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE: libx264 and libx265. Other
encoders will be supported in future commits.

doc/APIchanges

@@ -14,6 +14,9 @@ libavutil:     2021-04-27
 
 API changes, most recent first:
 
+2023-01-xx - xxxxxxxxxx - lavc 59.59.100 - avcodec.h
+  Add AV_CODEC_FLAG_COPY_OPAQUE.
+
 2023-01-13 - xxxxxxxxxx - lavu 57.44.100 - ambient_viewing_environment.h frame.h
   Adds a new structure for holding H.274 Ambient Viewing Environment metadata,
   AVAmbientViewingEnvironment.

libavcodec/avcodec.h

@@ -241,6 +241,32 @@ typedef struct RcOverride{
  * @ref AV_CODEC_CAP_ENCODER_RECON_FRAME capability.
  */
 #define AV_CODEC_FLAG_RECON_FRAME     (1 <<  6)
+/**
+ * Request the encoder to propagate each frame's AVFrame.opaque and
+ * AVFrame.opaque_ref values to its corresponding output AVPacket.
+ *
+ * May only be set on encoders that have the
+ * @ref AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE capability flag.
+ *
+ * @note
+ * While in typical cases one input frame produces exactly one output packet
+ * (perhaps after a delay), in general the mapping of frames to packets is
+ * M-to-N, so
+ * - Any number of input frames may be associated with any given output packet.
+ *   This includes zero - e.g. some encoders may output packets that carry only
+ *   metadata about the whole stream.
+ * - A given input frame may be associated with any number of output packets.
+ *   Again this includes zero - e.g. some encoders may drop frames under certain
+ *   conditions.
+ * .
+ * This implies that when using this flag, the caller must NOT assume that
+ * - a given input frame's opaques will necessarily appear on some output packet;
+ * - every output packet will have some non-NULL opaque value.
+ * .
+ * When an output packet contains multiple frames, the opaque values will be
+ * taken from the first of those.
+ */
+#define AV_CODEC_FLAG_COPY_OPAQUE     (1 <<  7)
 /**
  * Use internal 2pass ratecontrol in first pass mode.
  */

libavcodec/encode.c

@@ -636,6 +636,13 @@ int ff_encode_preinit(AVCodecContext *avctx)
         return AVERROR(EINVAL);
     }
 
+    if (avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE &&
+        !(avctx->codec->capabilities & AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE)) {
+        av_log(avctx, AV_LOG_ERROR, "The copy_opaque flag is set, but the "
+               "encoder does not support it.\n");
+        return AVERROR(EINVAL);
+    }
+
     switch (avctx->codec_type) {
     case AVMEDIA_TYPE_VIDEO: ret = encode_preinit_video(avctx); break;
     case AVMEDIA_TYPE_AUDIO: ret = encode_preinit_audio(avctx); break;

libavcodec/libx264.c

@@ -21,6 +21,7 @@
 
 #include "config_components.h"
 
+#include "libavutil/buffer.h"
 #include "libavutil/eval.h"
 #include "libavutil/internal.h"
 #include "libavutil/opt.h"
@@ -51,6 +52,9 @@
 typedef struct X264Opaque {
     int64_t reordered_opaque;
     int64_t wallclock;
+
+    void        *frame_opaque;
+    AVBufferRef *frame_opaque_ref;
 } X264Opaque;
 
 typedef struct X264Context {
@@ -133,6 +137,11 @@ static void X264_log(void *p, int level, const char *fmt, va_list args)
     av_vlog(p, level_map[level], fmt, args);
 }
 
+static void opaque_uninit(X264Opaque *o)
+{
+    av_buffer_unref(&o->frame_opaque_ref);
+    memset(o, 0, sizeof(*o));
+}
 
 static int encode_nals(AVCodecContext *ctx, AVPacket *pkt,
                        const x264_nal_t *nals, int nnal)
@@ -440,6 +449,15 @@ static int setup_frame(AVCodecContext *ctx, const AVFrame *frame,
 
     pic->i_pts  = frame->pts;
 
+    opaque_uninit(opaque);
+
+    if (ctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) {
+        opaque->frame_opaque = frame->opaque;
+        ret = av_buffer_replace(&opaque->frame_opaque_ref, frame->opaque_ref);
+        if (ret < 0)
+            goto fail;
+    }
+
     opaque->reordered_opaque = frame->reordered_opaque;
     opaque->wallclock = wallclock;
     if (ctx->export_side_data & AV_CODEC_EXPORT_DATA_PRFT)
@@ -594,6 +612,14 @@ static int X264_frame(AVCodecContext *ctx, AVPacket *pkt, const AVFrame *frame,
         out_opaque < &x4->reordered_opaque[x4->nb_reordered_opaque]) {
         ctx->reordered_opaque = out_opaque->reordered_opaque;
         wallclock = out_opaque->wallclock;
+
+        if (ctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) {
+            pkt->opaque                  = out_opaque->frame_opaque;
+            pkt->opaque_ref              = out_opaque->frame_opaque_ref;
+            out_opaque->frame_opaque_ref = NULL;
+        }
+
+        opaque_uninit(out_opaque);
     } else {
         // Unexpected opaque pointer on picture output
         av_log(ctx, AV_LOG_ERROR, "Unexpected opaque pointer; "
@@ -634,6 +660,9 @@ static av_cold int X264_close(AVCodecContext *avctx)
     X264Context *x4 = avctx->priv_data;
 
     av_freep(&x4->sei);
+
+    for (int i = 0; i < x4->nb_reordered_opaque; i++)
+        opaque_uninit(&x4->reordered_opaque[i]);
     av_freep(&x4->reordered_opaque);
 
 #if X264_BUILD >= 161

libavcodec/libx265.c

@@ -28,6 +28,7 @@
 #include <float.h>
 
 #include "libavutil/avassert.h"
+#include "libavutil/buffer.h"
 #include "libavutil/internal.h"
 #include "libavutil/common.h"
 #include "libavutil/opt.h"
@@ -43,6 +44,9 @@
 typedef struct ReorderedData {
     int64_t reordered_opaque;
 
+    void        *frame_opaque;
+    AVBufferRef *frame_opaque_ref;
+
     int in_use;
 } ReorderedData;
 
@@ -121,7 +125,7 @@ static int rd_get(libx265Context *ctx)
 static void rd_release(libx265Context *ctx, int idx)
 {
     av_assert0(idx >= 0 && idx < ctx->nb_rd);
-
+    av_buffer_unref(&ctx->rd[idx].frame_opaque_ref);
     memset(&ctx->rd[idx], 0, sizeof(ctx->rd[idx]));
 }
 
@@ -132,6 +136,8 @@ static av_cold int libx265_encode_close(AVCodecContext *avctx)
     ctx->api->param_free(ctx->params);
     av_freep(&ctx->sei_data);
 
+    for (int i = 0; i < ctx->nb_rd; i++)
+        rd_release(ctx, i);
     av_freep(&ctx->rd);
 
     if (ctx->encoder)
@@ -582,6 +588,9 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     sei->numPayloads = 0;
 
     if (pic) {
+        ReorderedData *rd;
+        int rd_idx;
+
         for (i = 0; i < 3; i++) {
            x265pic.planes[i] = pic->data[i];
            x265pic.stride[i] = pic->linesize[i];
@@ -600,21 +609,26 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
         if (ret < 0)
             return ret;
 
-        if (pic->reordered_opaque) {
+        rd_idx = rd_get(ctx);
-            ReorderedData *rd;
-            int rd_idx = rd_get(ctx);
-
         if (rd_idx < 0) {
             free_picture(ctx, &x265pic);
             return rd_idx;
         }
+        rd = &ctx->rd[rd_idx];
+
+        rd->reordered_opaque = pic->reordered_opaque;
+        if (avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) {
+            rd->frame_opaque = pic->opaque;
+            ret = av_buffer_replace(&rd->frame_opaque_ref, pic->opaque_ref);
+            if (ret < 0) {
+                rd_release(ctx, rd_idx);
+                free_picture(ctx, &x265pic);
+                return ret;
+            }
+        }
 
         x265pic.userData = (void*)(intptr_t)(rd_idx + 1);
 
-            rd = &ctx->rd[rd_idx];
-            rd->reordered_opaque = pic->reordered_opaque;
-        }
-
         if (ctx->a53_cc) {
             void *sei_data;
             size_t sei_size;
@@ -742,6 +756,12 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
 
         avctx->reordered_opaque = rd->reordered_opaque;
 
+        if (avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) {
+            pkt->opaque          = rd->frame_opaque;
+            pkt->opaque_ref      = rd->frame_opaque_ref;
+            rd->frame_opaque_ref = NULL;
+        }
+
         rd_release(ctx, idx);
     } else
         avctx->reordered_opaque = 0;

libavcodec/options_table.h

@@ -58,6 +58,7 @@ static const AVOption avcodec_options[] = {
 {"loop", "use loop filter", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_LOOP_FILTER }, INT_MIN, INT_MAX, V|E, "flags"},
 {"qscale", "use fixed qscale", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_QSCALE }, INT_MIN, INT_MAX, 0, "flags"},
 {"recon_frame", "export reconstructed frames", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_RECON_FRAME}, .unit = "flags"},
+{"copy_opaque", "propagate opaque values", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_COPY_OPAQUE}, .unit = "flags"},
 {"pass1", "use internal 2-pass ratecontrol in first  pass mode", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_PASS1 }, INT_MIN, INT_MAX, 0, "flags"},
 {"pass2", "use internal 2-pass ratecontrol in second pass mode", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_PASS2 }, INT_MIN, INT_MAX, 0, "flags"},
 {"gray", "only decode/encode grayscale", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_GRAY }, INT_MIN, INT_MAX, V|E|D, "flags"},

libavcodec/version.h

@@ -29,7 +29,7 @@
 
 #include "version_major.h"
 
-#define LIBAVCODEC_VERSION_MINOR  58
+#define LIBAVCODEC_VERSION_MINOR  59
 #define LIBAVCODEC_VERSION_MICRO 100
 
 #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \