diff --git a/libavcodec/av1dec.c b/libavcodec/av1dec.c
index 39ccad5bf6..87056520dd 100644
--- a/libavcodec/av1dec.c
+++ b/libavcodec/av1dec.c
@@ -37,6 +37,7 @@
 #include "internal.h"
 #include "hwconfig.h"
 #include "profiles.h"
+#include "refstruct.h"
 #include "thread.h"
 
 /**< same with Div_Lut defined in spec 7.11.3.7 */
@@ -641,7 +642,7 @@ static void av1_frame_unref(AVCodecContext *avctx, AV1Frame *f)
     ff_thread_release_buffer(avctx, f->f);
     av_buffer_unref(&f->hwaccel_priv_buf);
     f->hwaccel_picture_private = NULL;
-    av_buffer_unref(&f->header_ref);
+    ff_refstruct_unref(&f->header_ref);
     f->raw_frame_header = NULL;
     f->spatial_id = f->temporal_id = 0;
     memset(f->skip_mode_frame_idx, 0,
@@ -654,9 +655,7 @@ static int av1_frame_ref(AVCodecContext *avctx, AV1Frame *dst, const AV1Frame *s
 {
     int ret;
 
-    ret = av_buffer_replace(&dst->header_ref, src->header_ref);
-    if (ret < 0)
-        return ret;
+    ff_refstruct_replace(&dst->header_ref, src->header_ref);
 
     dst->raw_frame_header = src->raw_frame_header;
 
@@ -712,10 +711,10 @@ static av_cold int av1_decode_free(AVCodecContext *avctx)
     av1_frame_unref(avctx, &s->cur_frame);
     av_frame_free(&s->cur_frame.f);
 
-    av_buffer_unref(&s->seq_ref);
-    av_buffer_unref(&s->header_ref);
-    av_buffer_unref(&s->cll_ref);
-    av_buffer_unref(&s->mdcv_ref);
+    ff_refstruct_unref(&s->seq_ref);
+    ff_refstruct_unref(&s->header_ref);
+    ff_refstruct_unref(&s->cll_ref);
+    ff_refstruct_unref(&s->mdcv_ref);
     av_freep(&s->tile_group_info);
 
     while (s->itut_t35_fifo && av_fifo_read(s->itut_t35_fifo, &itut_t35, 1) >= 0)
@@ -1160,9 +1159,7 @@ static int get_current_frame(AVCodecContext *avctx)
 
     av1_frame_unref(avctx, &s->cur_frame);
 
-    s->cur_frame.header_ref = av_buffer_ref(s->header_ref);
-    if (!s->cur_frame.header_ref)
-        return AVERROR(ENOMEM);
+    s->cur_frame.header_ref = ff_refstruct_ref(s->header_ref);
 
     s->cur_frame.raw_frame_header = s->raw_frame_header;
 
@@ -1214,12 +1211,7 @@ static int av1_receive_frame_internal(AVCodecContext *avctx, AVFrame *frame)
 
         switch (unit->type) {
         case AV1_OBU_SEQUENCE_HEADER:
-            av_buffer_unref(&s->seq_ref);
-            s->seq_ref = av_buffer_ref(unit->content_ref);
-            if (!s->seq_ref) {
-                ret = AVERROR(ENOMEM);
-                goto end;
-            }
+            ff_refstruct_replace(&s->seq_ref, unit->content_ref);
 
             s->raw_seq = &obu->obu.sequence_header;
 
@@ -1264,12 +1256,7 @@ static int av1_receive_frame_internal(AVCodecContext *avctx, AVFrame *frame)
                 goto end;
             }
 
-            av_buffer_unref(&s->header_ref);
-            s->header_ref = av_buffer_ref(unit->content_ref);
-            if (!s->header_ref) {
-                ret = AVERROR(ENOMEM);
-                goto end;
-            }
+            ff_refstruct_replace(&s->header_ref, unit->content_ref);
 
             if (unit->type == AV1_OBU_FRAME)
                 s->raw_frame_header = &obu->obu.frame.header;
@@ -1356,23 +1343,11 @@ static int av1_receive_frame_internal(AVCodecContext *avctx, AVFrame *frame)
         case AV1_OBU_METADATA:
             switch (obu->obu.metadata.metadata_type) {
             case AV1_METADATA_TYPE_HDR_CLL:
-                av_buffer_unref(&s->cll_ref);
-                s->cll_ref = av_buffer_ref(unit->content_ref);
-                if (!s->cll_ref) {
-                    s->cll = NULL;
-                    ret = AVERROR(ENOMEM);
-                    goto end;
-                }
+                ff_refstruct_replace(&s->cll_ref, unit->content_ref);
                 s->cll = &obu->obu.metadata.metadata.hdr_cll;
                 break;
             case AV1_METADATA_TYPE_HDR_MDCV:
-                av_buffer_unref(&s->mdcv_ref);
-                s->mdcv_ref = av_buffer_ref(unit->content_ref);
-                if (!s->mdcv_ref) {
-                    s->mdcv = NULL;
-                    ret = AVERROR(ENOMEM);
-                    goto end;
-                }
+                ff_refstruct_replace(&s->mdcv_ref, unit->content_ref);
                 s->mdcv = &obu->obu.metadata.metadata.hdr_mdcv;
                 break;
             case AV1_METADATA_TYPE_ITUT_T35: {
diff --git a/libavcodec/av1dec.h b/libavcodec/av1dec.h
index 59ffed1d9b..acbeec4af3 100644
--- a/libavcodec/av1dec.h
+++ b/libavcodec/av1dec.h
@@ -38,7 +38,7 @@ typedef struct AV1Frame {
     AVBufferRef *hwaccel_priv_buf;
     void *hwaccel_picture_private;
 
-    AVBufferRef *header_ref;
+    AV1RawOBU *header_ref; ///< RefStruct reference backing raw_frame_header.
     AV1RawFrameHeader *raw_frame_header;
 
     int temporal_id;
@@ -71,15 +71,15 @@ typedef struct AV1DecContext {
     CodedBitstreamFragment current_obu;
     AVPacket *pkt;
 
-    AVBufferRef *seq_ref;
+    AV1RawOBU *seq_ref;    ///< RefStruct reference backing raw_seq
     AV1RawSequenceHeader *raw_seq;
-    AVBufferRef *header_ref;
+    AV1RawOBU *header_ref; ///< RefStruct reference backing raw_frame_header
     AV1RawFrameHeader *raw_frame_header;
     TileGroupInfo *tile_group_info;
 
-    AVBufferRef *cll_ref;
+    AV1RawOBU *cll_ref;    ///< RefStruct reference backing cll
     AV1RawMetadataHDRCLL *cll;
-    AVBufferRef *mdcv_ref;
+    AV1RawOBU *mdcv_ref;   ///< RefStruct reference backing mdcv
     AV1RawMetadataHDRMDCV *mdcv;
     AVFifo *itut_t35_fifo;
 
diff --git a/libavcodec/cbs.c b/libavcodec/cbs.c
index daf7f66427..cdd7adebeb 100644
--- a/libavcodec/cbs.c
+++ b/libavcodec/cbs.c
@@ -28,6 +28,7 @@
 #include "avcodec.h"
 #include "cbs.h"
 #include "cbs_internal.h"
+#include "refstruct.h"
 
 
 static const CodedBitstreamType *const cbs_type_table[] = {
@@ -152,7 +153,7 @@ av_cold void ff_cbs_close(CodedBitstreamContext **ctx_ptr)
 
 static void cbs_unit_uninit(CodedBitstreamUnit *unit)
 {
-    av_buffer_unref(&unit->content_ref);
+    ff_refstruct_unref(&unit->content_ref);
     unit->content = NULL;
 
     av_buffer_unref(&unit->data_ref);
@@ -200,7 +201,7 @@ static int cbs_read_fragment_content(CodedBitstreamContext *ctx,
                 continue;
         }
 
-        av_buffer_unref(&unit->content_ref);
+        ff_refstruct_unref(&unit->content_ref);
         unit->content = NULL;
 
         av_assert0(unit->data && unit->data_ref);
@@ -214,7 +215,7 @@ static int cbs_read_fragment_content(CodedBitstreamContext *ctx,
             av_log(ctx->log_ctx, AV_LOG_VERBOSE,
                    "Skipping decomposition of unit %d "
                    "(type %"PRIu32").\n", i, unit->type);
-            av_buffer_unref(&unit->content_ref);
+            ff_refstruct_unref(&unit->content_ref);
             unit->content = NULL;
         } else if (err < 0) {
             av_log(ctx->log_ctx, AV_LOG_ERROR, "Failed to read unit %d "
@@ -776,28 +777,22 @@ int ff_cbs_insert_unit_content(CodedBitstreamFragment *frag,
                                int position,
                                CodedBitstreamUnitType type,
                                void *content,
-                               AVBufferRef *content_buf)
+                               void *content_ref)
 {
     CodedBitstreamUnit *unit;
-    AVBufferRef *content_ref;
     int err;
 
     if (position == -1)
         position = frag->nb_units;
     av_assert0(position >= 0 && position <= frag->nb_units);
 
-    if (content_buf) {
-        content_ref = av_buffer_ref(content_buf);
-        if (!content_ref)
-            return AVERROR(ENOMEM);
-    } else {
-        content_ref = NULL;
-    }
-
     err = cbs_insert_unit(frag, position);
-    if (err < 0) {
-        av_buffer_unref(&content_ref);
+    if (err < 0)
         return err;
+
+    if (content_ref) {
+        // Create our own reference out of the user-supplied one.
+        content_ref = ff_refstruct_ref(content_ref);
     }
 
     unit = &frag->units[position];
@@ -871,15 +866,14 @@ void ff_cbs_delete_unit(CodedBitstreamFragment *frag,
                 (frag->nb_units - position) * sizeof(*frag->units));
 }
 
-static void cbs_default_free_unit_content(void *opaque, uint8_t *data)
+static void cbs_default_free_unit_content(FFRefStructOpaque opaque, void *content)
 {
-    const CodedBitstreamUnitTypeDescriptor *desc = opaque;
+    const CodedBitstreamUnitTypeDescriptor *desc = opaque.c;
 
     for (int i = 0; i < desc->type.ref.nb_offsets; i++) {
-        void **ptr = (void**)(data + desc->type.ref.offsets[i]);
+        void **ptr = (void**)((char*)content + desc->type.ref.offsets[i]);
         av_buffer_unref((AVBufferRef**)(ptr + 1));
     }
-    av_free(data);
 }
 
 static const CodedBitstreamUnitTypeDescriptor
@@ -910,6 +904,15 @@ static const CodedBitstreamUnitTypeDescriptor
     return NULL;
 }
 
+static void *cbs_alloc_content(const CodedBitstreamUnitTypeDescriptor *desc)
+{
+    return ff_refstruct_alloc_ext_c(desc->content_size, 0,
+                                    (FFRefStructOpaque){ .c = desc },
+                                    desc->content_type == CBS_CONTENT_TYPE_COMPLEX
+                                            ? desc->type.complex.content_free
+                                            : cbs_default_free_unit_content);
+}
+
 int ff_cbs_alloc_unit_content(CodedBitstreamContext *ctx,
                               CodedBitstreamUnit *unit)
 {
@@ -921,27 +924,17 @@ int ff_cbs_alloc_unit_content(CodedBitstreamContext *ctx,
     if (!desc)
         return AVERROR(ENOSYS);
 
-    unit->content = av_mallocz(desc->content_size);
-    if (!unit->content)
+    unit->content_ref = cbs_alloc_content(desc);
+    if (!unit->content_ref)
         return AVERROR(ENOMEM);
-
-    unit->content_ref =
-        av_buffer_create(unit->content, desc->content_size,
-                         desc->content_type == CBS_CONTENT_TYPE_COMPLEX
-                                            ? desc->type.complex.content_free
-                                            : cbs_default_free_unit_content,
-                         (void*)desc, 0);
-    if (!unit->content_ref) {
-        av_freep(&unit->content);
-        return AVERROR(ENOMEM);
-    }
+    unit->content = unit->content_ref;
 
     return 0;
 }
 
-static int cbs_clone_internal_refs_unit_content(AVBufferRef **clone_ref,
-                                                const CodedBitstreamUnit *unit,
-                                                const CodedBitstreamUnitTypeDescriptor *desc)
+static int cbs_clone_noncomplex_unit_content(void **clonep,
+                                             const CodedBitstreamUnit *unit,
+                                             const CodedBitstreamUnitTypeDescriptor *desc)
 {
     const uint8_t *src;
     uint8_t *copy;
@@ -950,9 +943,15 @@ static int cbs_clone_internal_refs_unit_content(AVBufferRef **clone_ref,
     av_assert0(unit->content);
     src = unit->content;
 
-    copy = av_memdup(src, desc->content_size);
+    copy = cbs_alloc_content(desc);
     if (!copy)
         return AVERROR(ENOMEM);
+    memcpy(copy, src, desc->content_size);
+    for (int i = 0; i < desc->type.ref.nb_offsets; i++) {
+        void **ptr = (void**)(copy + desc->type.ref.offsets[i]);
+        /* Zero all the AVBufferRefs as they are owned by src. */
+        *(ptr + 1) = NULL;
+    }
 
     for (i = 0; i < desc->type.ref.nb_offsets; i++) {
         const uint8_t *const *src_ptr = (const uint8_t* const*)(src + desc->type.ref.offsets[i]);
@@ -978,22 +977,12 @@ static int cbs_clone_internal_refs_unit_content(AVBufferRef **clone_ref,
             goto fail;
         }
     }
-
-    *clone_ref = av_buffer_create(copy, desc->content_size,
-                                  cbs_default_free_unit_content,
-                                  (void*)desc, 0);
-    if (!*clone_ref) {
-        err = AVERROR(ENOMEM);
-        goto fail;
-    }
+    *clonep = copy;
 
     return 0;
 
 fail:
-    for (--i; i >= 0; i--)
-        av_buffer_unref((AVBufferRef**)(copy + desc->type.ref.offsets[i]));
-    av_freep(&copy);
-    *clone_ref = NULL;
+    ff_refstruct_unref(&copy);
     return err;
 }
 
@@ -1006,7 +995,7 @@ static int cbs_clone_unit_content(CodedBitstreamContext *ctx,
                                   CodedBitstreamUnit *unit)
 {
     const CodedBitstreamUnitTypeDescriptor *desc;
-    AVBufferRef *ref;
+    void *new_content;
     int err;
 
     desc = cbs_find_unit_type_desc(ctx, unit);
@@ -1015,13 +1004,13 @@ static int cbs_clone_unit_content(CodedBitstreamContext *ctx,
 
     switch (desc->content_type) {
     case CBS_CONTENT_TYPE_INTERNAL_REFS:
-        err = cbs_clone_internal_refs_unit_content(&ref, unit, desc);
+        err = cbs_clone_noncomplex_unit_content(&new_content, unit, desc);
         break;
 
     case CBS_CONTENT_TYPE_COMPLEX:
         if (!desc->type.complex.content_clone)
             return AVERROR_PATCHWELCOME;
-        err = desc->type.complex.content_clone(&ref, unit);
+        err = desc->type.complex.content_clone(&new_content, unit);
         break;
 
     default:
@@ -1031,8 +1020,8 @@ static int cbs_clone_unit_content(CodedBitstreamContext *ctx,
     if (err < 0)
         return err;
 
-    unit->content_ref = ref;
-    unit->content     = ref->data;
+    unit->content_ref = new_content;
+    unit->content     = new_content;
     return 0;
 }
 
@@ -1048,17 +1037,17 @@ int ff_cbs_make_unit_refcounted(CodedBitstreamContext *ctx,
 int ff_cbs_make_unit_writable(CodedBitstreamContext *ctx,
                               CodedBitstreamUnit *unit)
 {
-    AVBufferRef *ref = unit->content_ref;
+    void *ref = unit->content_ref;
     int err;
 
     av_assert0(unit->content);
-    if (ref && av_buffer_is_writable(ref))
+    if (ref && ff_refstruct_exclusive(ref))
         return 0;
 
     err = cbs_clone_unit_content(ctx, unit);
     if (err < 0)
         return err;
-    av_buffer_unref(&ref);
+    ff_refstruct_unref(&ref);
     return 0;
 }
 
diff --git a/libavcodec/cbs.h b/libavcodec/cbs.h
index ffb2797761..d479b1ac2d 100644
--- a/libavcodec/cbs.h
+++ b/libavcodec/cbs.h
@@ -106,10 +106,10 @@ typedef struct CodedBitstreamUnit {
      */
     void *content;
     /**
-     * If content is reference counted, a reference to the buffer containing
-     * content.  Null if content is not reference counted.
+     * If content is reference counted, a RefStruct reference backing content.
+     * NULL if content is not reference counted.
      */
-    AVBufferRef *content_ref;
+    void *content_ref;
 } CodedBitstreamUnit;
 
 /**
@@ -438,14 +438,16 @@ int ff_cbs_alloc_unit_content(CodedBitstreamContext *ctx,
 /**
  * Insert a new unit into a fragment with the given content.
  *
+ * If content_ref is supplied, it has to be a RefStruct reference
+ * backing content; the user keeps ownership of the supplied reference.
  * The content structure continues to be owned by the caller if
- * content_buf is not supplied.
+ * content_ref is not supplied.
  */
 int ff_cbs_insert_unit_content(CodedBitstreamFragment *frag,
                                int position,
                                CodedBitstreamUnitType type,
                                void *content,
-                               AVBufferRef *content_buf);
+                               void *content_ref);
 
 /**
  * Add a new unit to a fragment with the given data bitstream.
diff --git a/libavcodec/cbs_av1.c b/libavcodec/cbs_av1.c
index ed9a7b80d4..1d9ac5ab44 100644
--- a/libavcodec/cbs_av1.c
+++ b/libavcodec/cbs_av1.c
@@ -24,6 +24,7 @@
 #include "cbs_internal.h"
 #include "cbs_av1.h"
 #include "defs.h"
+#include "refstruct.h"
 
 
 static int cbs_av1_read_uvlc(CodedBitstreamContext *ctx, GetBitContext *gbc,
@@ -869,12 +870,7 @@ static int cbs_av1_read_unit(CodedBitstreamContext *ctx,
                 priv->operating_point_idc = sequence_header->operating_point_idc[priv->operating_point];
             }
 
-            av_buffer_unref(&priv->sequence_header_ref);
-            priv->sequence_header = NULL;
-
-            priv->sequence_header_ref = av_buffer_ref(unit->content_ref);
-            if (!priv->sequence_header_ref)
-                return AVERROR(ENOMEM);
+            ff_refstruct_replace(&priv->sequence_header_ref, unit->content_ref);
             priv->sequence_header = &obu->obu.sequence_header;
         }
         break;
@@ -993,9 +989,7 @@ static int cbs_av1_write_obu(CodedBitstreamContext *ctx,
     av1ctx = *priv;
 
     if (priv->sequence_header_ref) {
-        av1ctx.sequence_header_ref = av_buffer_ref(priv->sequence_header_ref);
-        if (!av1ctx.sequence_header_ref)
-            return AVERROR(ENOMEM);
+        av1ctx.sequence_header_ref = ff_refstruct_ref(priv->sequence_header_ref);
     }
 
     if (priv->frame_header_ref) {
@@ -1033,19 +1027,14 @@ static int cbs_av1_write_obu(CodedBitstreamContext *ctx,
             if (err < 0)
                 goto error;
 
-            av_buffer_unref(&priv->sequence_header_ref);
+            ff_refstruct_unref(&priv->sequence_header_ref);
             priv->sequence_header = NULL;
 
             err = ff_cbs_make_unit_refcounted(ctx, unit);
             if (err < 0)
                 goto error;
 
-            priv->sequence_header_ref = av_buffer_ref(unit->content_ref);
-            if (!priv->sequence_header_ref) {
-                err = AVERROR(ENOMEM);
-                goto error;
-            }
-
+            priv->sequence_header_ref = ff_refstruct_ref(unit->content_ref);
             priv->sequence_header = &obu->obu.sequence_header;
         }
         break;
@@ -1149,7 +1138,7 @@ static int cbs_av1_write_obu(CodedBitstreamContext *ctx,
     av_assert0(data_pos <= start_pos);
 
     if (8 * obu->obu_size > put_bits_left(pbc)) {
-        av_buffer_unref(&priv->sequence_header_ref);
+        ff_refstruct_unref(&priv->sequence_header_ref);
         av_buffer_unref(&priv->frame_header_ref);
         *priv = av1ctx;
 
@@ -1178,7 +1167,7 @@ static int cbs_av1_write_obu(CodedBitstreamContext *ctx,
     err = 0;
 
 error:
-    av_buffer_unref(&av1ctx.sequence_header_ref);
+    ff_refstruct_unref(&av1ctx.sequence_header_ref);
     av_buffer_unref(&av1ctx.frame_header_ref);
 
     return err;
@@ -1230,13 +1219,13 @@ static void cbs_av1_close(CodedBitstreamContext *ctx)
 {
     CodedBitstreamAV1Context *priv = ctx->priv_data;
 
-    av_buffer_unref(&priv->sequence_header_ref);
+    ff_refstruct_unref(&priv->sequence_header_ref);
     av_buffer_unref(&priv->frame_header_ref);
 }
 
-static void cbs_av1_free_metadata(void *unit, uint8_t *content)
+static void cbs_av1_free_metadata(FFRefStructOpaque unused, void *content)
 {
-    AV1RawOBU *obu = (AV1RawOBU*)content;
+    AV1RawOBU *obu = content;
     AV1RawMetadata *md;
 
     av_assert0(obu->header.obu_type == AV1_OBU_METADATA);
@@ -1254,7 +1243,6 @@ static void cbs_av1_free_metadata(void *unit, uint8_t *content)
     default:
         av_buffer_unref(&md->metadata.unknown.payload_ref);
     }
-    av_free(content);
 }
 
 static const CodedBitstreamUnitTypeDescriptor cbs_av1_unit_types[] = {
diff --git a/libavcodec/cbs_av1.h b/libavcodec/cbs_av1.h
index 7924257164..a5402f069d 100644
--- a/libavcodec/cbs_av1.h
+++ b/libavcodec/cbs_av1.h
@@ -437,7 +437,8 @@ typedef struct CodedBitstreamAV1Context {
     const AVClass *class;
 
     AV1RawSequenceHeader *sequence_header;
-    AVBufferRef          *sequence_header_ref;
+    /** A RefStruct reference backing sequence_header. */
+    AV1RawOBU            *sequence_header_ref;
 
     int     seen_frame_header;
     AVBufferRef *frame_header_ref;
diff --git a/libavcodec/cbs_h264.h b/libavcodec/cbs_h264.h
index ca9b688c05..db91231337 100644
--- a/libavcodec/cbs_h264.h
+++ b/libavcodec/cbs_h264.h
@@ -407,10 +407,8 @@ typedef struct CodedBitstreamH264Context {
 
     // All currently available parameter sets.  These are updated when
     // any parameter set NAL unit is read/written with this context.
-    AVBufferRef *sps_ref[H264_MAX_SPS_COUNT];
-    AVBufferRef *pps_ref[H264_MAX_PPS_COUNT];
-    H264RawSPS *sps[H264_MAX_SPS_COUNT];
-    H264RawPPS *pps[H264_MAX_PPS_COUNT];
+    H264RawSPS *sps[H264_MAX_SPS_COUNT]; ///< RefStruct references
+    H264RawPPS *pps[H264_MAX_PPS_COUNT]; ///< RefStruct references
 
     // The currently active parameter sets.  These are updated when any
     // NAL unit refers to the relevant parameter set.  These pointers
diff --git a/libavcodec/cbs_h2645.c b/libavcodec/cbs_h2645.c
index 0a1c8ea426..cce3f31d4f 100644
--- a/libavcodec/cbs_h2645.c
+++ b/libavcodec/cbs_h2645.c
@@ -28,6 +28,7 @@
 #include "h264.h"
 #include "h2645_parse.h"
 #include "hevc.h"
+#include "refstruct.h"
 #include "vvc.h"
 
 
@@ -733,12 +734,8 @@ static int cbs_h26 ## h26n ## _replace_ ## ps_var(CodedBitstreamContext *ctx, \
         return err; \
     if (priv->ps_var[id] == priv->active_ ## ps_var) \
         priv->active_ ## ps_var = NULL ; \
-    av_buffer_unref(&priv->ps_var ## _ref[id]); \
     av_assert0(unit->content_ref); \
-    priv->ps_var ## _ref[id] = av_buffer_ref(unit->content_ref); \
-    if (!priv->ps_var ## _ref[id]) \
-        return AVERROR(ENOMEM); \
-    priv->ps_var[id] = (H26 ## h26n ## Raw ## ps_name *)priv->ps_var ## _ref[id]->data; \
+    ff_refstruct_replace(&priv->ps_var[id], unit->content_ref); \
     return 0; \
 }
 
@@ -758,12 +755,8 @@ static int cbs_h26 ## h26n ## _replace_ ## ps_var(CodedBitstreamContext *ctx, \
     int err = ff_cbs_make_unit_refcounted(ctx, unit); \
     if (err < 0) \
         return err; \
-    av_buffer_unref(&priv->ps_var ## _ref[id]); \
     av_assert0(unit->content_ref); \
-    priv->ps_var ## _ref[id] = av_buffer_ref(unit->content_ref); \
-    if (!priv->ps_var ## _ref[id]) \
-        return AVERROR(ENOMEM); \
-    priv->ps_var[id] = (H26 ## h26n ## Raw ## ps_name *)priv->ps_var ## _ref[id]->data; \
+    ff_refstruct_replace(&priv->ps_var[id], unit->content_ref); \
     return 0; \
 }
 
@@ -782,9 +775,7 @@ static int cbs_h266_replace_ph(CodedBitstreamContext *ctx,
     if (err < 0)
         return err;
     av_assert0(unit->content_ref);
-    err = av_buffer_replace(&h266->ph_ref, unit->content_ref);
-    if (err < 0)
-        return err;
+    ff_refstruct_replace(&h266->ph_ref, unit->content_ref);
     h266->ph = ph;
     return 0;
 }
@@ -1848,14 +1839,10 @@ static void cbs_h264_flush(CodedBitstreamContext *ctx)
 {
     CodedBitstreamH264Context *h264 = ctx->priv_data;
 
-    for (int i = 0; i < FF_ARRAY_ELEMS(h264->sps); i++) {
-        av_buffer_unref(&h264->sps_ref[i]);
-        h264->sps[i] = NULL;
-    }
-    for (int i = 0; i < FF_ARRAY_ELEMS(h264->pps); i++) {
-        av_buffer_unref(&h264->pps_ref[i]);
-        h264->pps[i] = NULL;
-    }
+    for (int i = 0; i < FF_ARRAY_ELEMS(h264->sps); i++)
+        ff_refstruct_unref(&h264->sps[i]);
+    for (int i = 0; i < FF_ARRAY_ELEMS(h264->pps); i++)
+        ff_refstruct_unref(&h264->pps[i]);
 
     h264->active_sps = NULL;
     h264->active_pps = NULL;
@@ -1870,27 +1857,21 @@ static void cbs_h264_close(CodedBitstreamContext *ctx)
     ff_h2645_packet_uninit(&h264->common.read_packet);
 
     for (i = 0; i < FF_ARRAY_ELEMS(h264->sps); i++)
-        av_buffer_unref(&h264->sps_ref[i]);
+        ff_refstruct_unref(&h264->sps[i]);
     for (i = 0; i < FF_ARRAY_ELEMS(h264->pps); i++)
-        av_buffer_unref(&h264->pps_ref[i]);
+        ff_refstruct_unref(&h264->pps[i]);
 }
 
 static void cbs_h265_flush(CodedBitstreamContext *ctx)
 {
     CodedBitstreamH265Context *h265 = ctx->priv_data;
 
-    for (int i = 0; i < FF_ARRAY_ELEMS(h265->vps); i++) {
-        av_buffer_unref(&h265->vps_ref[i]);
-        h265->vps[i] = NULL;
-    }
-    for (int i = 0; i < FF_ARRAY_ELEMS(h265->sps); i++) {
-        av_buffer_unref(&h265->sps_ref[i]);
-        h265->sps[i] = NULL;
-    }
-    for (int i = 0; i < FF_ARRAY_ELEMS(h265->pps); i++) {
-        av_buffer_unref(&h265->pps_ref[i]);
-        h265->pps[i] = NULL;
-    }
+    for (int i = 0; i < FF_ARRAY_ELEMS(h265->vps); i++)
+        ff_refstruct_unref(&h265->vps[i]);
+    for (int i = 0; i < FF_ARRAY_ELEMS(h265->sps); i++)
+        ff_refstruct_unref(&h265->sps[i]);
+    for (int i = 0; i < FF_ARRAY_ELEMS(h265->pps); i++)
+        ff_refstruct_unref(&h265->pps[i]);
 
     h265->active_vps = NULL;
     h265->active_sps = NULL;
@@ -1905,32 +1886,24 @@ static void cbs_h265_close(CodedBitstreamContext *ctx)
     ff_h2645_packet_uninit(&h265->common.read_packet);
 
     for (i = 0; i < FF_ARRAY_ELEMS(h265->vps); i++)
-        av_buffer_unref(&h265->vps_ref[i]);
+        ff_refstruct_unref(&h265->vps[i]);
     for (i = 0; i < FF_ARRAY_ELEMS(h265->sps); i++)
-        av_buffer_unref(&h265->sps_ref[i]);
+        ff_refstruct_unref(&h265->sps[i]);
     for (i = 0; i < FF_ARRAY_ELEMS(h265->pps); i++)
-        av_buffer_unref(&h265->pps_ref[i]);
+        ff_refstruct_unref(&h265->pps[i]);
 }
 
 static void cbs_h266_flush(CodedBitstreamContext *ctx)
 {
     CodedBitstreamH266Context *h266 = ctx->priv_data;
 
-    for (int i = 0; i < FF_ARRAY_ELEMS(h266->vps); i++) {
-        av_buffer_unref(&h266->vps_ref[i]);
-        h266->vps[i] = NULL;
-    }
-
-    for (int i = 0; i < FF_ARRAY_ELEMS(h266->sps); i++) {
-        av_buffer_unref(&h266->sps_ref[i]);
-        h266->sps[i] = NULL;
-    }
-    for (int i = 0; i < FF_ARRAY_ELEMS(h266->pps); i++) {
-        av_buffer_unref(&h266->pps_ref[i]);
-        h266->pps[i] = NULL;
-    }
-    av_buffer_unref(&h266->ph_ref);
-    h266->ph = NULL;
+    for (int i = 0; i < FF_ARRAY_ELEMS(h266->vps); i++)
+        ff_refstruct_unref(&h266->vps[i]);
+    for (int i = 0; i < FF_ARRAY_ELEMS(h266->sps); i++)
+        ff_refstruct_unref(&h266->sps[i]);
+    for (int i = 0; i < FF_ARRAY_ELEMS(h266->pps); i++)
+        ff_refstruct_unref(&h266->pps[i]);
+    ff_refstruct_unref(&h266->ph_ref);
 }
 
 static void cbs_h266_close(CodedBitstreamContext *ctx)
@@ -1941,11 +1914,10 @@ static void cbs_h266_close(CodedBitstreamContext *ctx)
     ff_h2645_packet_uninit(&h266->common.read_packet);
  }
 
-static void cbs_h264_free_sei(void *opaque, uint8_t *content)
+static void cbs_h264_free_sei(FFRefStructOpaque unused, void *content)
 {
-    H264RawSEI *sei = (H264RawSEI*)content;
+    H264RawSEI *sei = content;
     ff_cbs_sei_free_message_list(&sei->message_list);
-    av_free(content);
 }
 
 static const CodedBitstreamUnitTypeDescriptor cbs_h264_unit_types[] = {
@@ -1968,11 +1940,10 @@ static const CodedBitstreamUnitTypeDescriptor cbs_h264_unit_types[] = {
     CBS_UNIT_TYPE_END_OF_LIST
 };
 
-static void cbs_h265_free_sei(void *opaque, uint8_t *content)
+static void cbs_h265_free_sei(FFRefStructOpaque unused, void *content)
 {
-    H265RawSEI *sei = (H265RawSEI*)content;
+    H265RawSEI *sei = content;
     ff_cbs_sei_free_message_list(&sei->message_list);
-    av_free(content);
 }
 
 static const CodedBitstreamUnitTypeDescriptor cbs_h265_unit_types[] = {
@@ -1995,11 +1966,10 @@ static const CodedBitstreamUnitTypeDescriptor cbs_h265_unit_types[] = {
     CBS_UNIT_TYPE_END_OF_LIST
 };
 
-static void cbs_h266_free_sei(void *opaque, uint8_t *content)
+static void cbs_h266_free_sei(FFRefStructOpaque unused, void *content)
 {
-    H266RawSEI *sei = (H266RawSEI*)content;
+    H266RawSEI *sei = content;
     ff_cbs_sei_free_message_list(&sei->message_list);
-    av_free(content);
 }
 
 static const CodedBitstreamUnitTypeDescriptor cbs_h266_unit_types[] = {
diff --git a/libavcodec/cbs_h265.h b/libavcodec/cbs_h265.h
index f7cbd4970d..1b1195f198 100644
--- a/libavcodec/cbs_h265.h
+++ b/libavcodec/cbs_h265.h
@@ -681,12 +681,9 @@ typedef struct CodedBitstreamH265Context {
 
     // All currently available parameter sets.  These are updated when
     // any parameter set NAL unit is read/written with this context.
-    AVBufferRef *vps_ref[HEVC_MAX_VPS_COUNT];
-    AVBufferRef *sps_ref[HEVC_MAX_SPS_COUNT];
-    AVBufferRef *pps_ref[HEVC_MAX_PPS_COUNT];
-    H265RawVPS *vps[HEVC_MAX_VPS_COUNT];
-    H265RawSPS *sps[HEVC_MAX_SPS_COUNT];
-    H265RawPPS *pps[HEVC_MAX_PPS_COUNT];
+    H265RawVPS *vps[HEVC_MAX_VPS_COUNT]; ///< RefStruct references
+    H265RawSPS *sps[HEVC_MAX_SPS_COUNT]; ///< RefStruct references
+    H265RawPPS *pps[HEVC_MAX_PPS_COUNT]; ///< RefStruct references
 
     // The currently active parameter sets.  These are updated when any
     // NAL unit refers to the relevant parameter set.  These pointers
diff --git a/libavcodec/cbs_h266.h b/libavcodec/cbs_h266.h
index 3a6f6d96b5..d2ba99f522 100644
--- a/libavcodec/cbs_h266.h
+++ b/libavcodec/cbs_h266.h
@@ -867,14 +867,11 @@ typedef struct CodedBitstreamH266Context {
 
     // All currently available parameter sets.  These are updated when
     // any parameter set NAL unit is read/written with this context.
-    AVBufferRef *vps_ref[VVC_MAX_VPS_COUNT];
-    AVBufferRef *sps_ref[VVC_MAX_SPS_COUNT];
-    AVBufferRef *pps_ref[VVC_MAX_PPS_COUNT];
-    AVBufferRef *ph_ref;
-    H266RawVPS  *vps[VVC_MAX_VPS_COUNT];
-    H266RawSPS  *sps[VVC_MAX_SPS_COUNT];
-    H266RawPPS  *pps[VVC_MAX_PPS_COUNT];
+    H266RawVPS  *vps[VVC_MAX_VPS_COUNT]; ///< RefStruct references
+    H266RawSPS  *sps[VVC_MAX_SPS_COUNT]; ///< RefStruct references
+    H266RawPPS  *pps[VVC_MAX_PPS_COUNT]; ///< RefStruct references
     H266RawPictureHeader *ph;
+    void *ph_ref; ///< RefStruct reference backing ph above
 } CodedBitstreamH266Context;
 
 #endif /* AVCODEC_CBS_H266_H */
diff --git a/libavcodec/cbs_h266_syntax_template.c b/libavcodec/cbs_h266_syntax_template.c
index aecc4d79f8..5654f22878 100644
--- a/libavcodec/cbs_h266_syntax_template.c
+++ b/libavcodec/cbs_h266_syntax_template.c
@@ -1074,17 +1074,13 @@ static int FUNC(sps)(CodedBitstreamContext *ctx, RWContext *rw,
 
     ub(4, sps_seq_parameter_set_id);
     ub(4, sps_video_parameter_set_id);
-    if (current->sps_video_parameter_set_id == 0 && !h266->vps_ref[0]) {
-        H266RawVPS *vps;
-        AVBufferRef *ref = av_buffer_allocz(sizeof(H266RawVPS));
-        if (!ref) {
+    if (current->sps_video_parameter_set_id == 0 && !h266->vps[0]) {
+        H266RawVPS *vps = ff_refstruct_allocz(sizeof(*vps));
+        if (!vps)
             return AVERROR(ENOMEM);
-        }
-        vps = (H266RawVPS *) ref->data;
         vps->vps_max_layers_minus1 = 0;
         vps->vps_independent_layer_flag[0] = 1;
         vps->vps_layer_id[0] = current->nal_unit_header.nuh_layer_id;
-        h266->vps_ref[0] = ref;
         h266->vps[0] = vps;
     }
 
diff --git a/libavcodec/cbs_internal.h b/libavcodec/cbs_internal.h
index 285deb40d5..07220f1f3e 100644
--- a/libavcodec/cbs_internal.h
+++ b/libavcodec/cbs_internal.h
@@ -19,15 +19,16 @@
 #ifndef AVCODEC_CBS_INTERNAL_H
 #define AVCODEC_CBS_INTERNAL_H
 
+#include <stddef.h>
 #include <stdint.h>
 
-#include "libavutil/buffer.h"
 #include "libavutil/log.h"
 
 #include "cbs.h"
 #include "codec_id.h"
 #include "get_bits.h"
 #include "put_bits.h"
+#include "refstruct.h"
 
 
 enum CBSContentType {
@@ -92,8 +93,8 @@ typedef const struct CodedBitstreamUnitTypeDescriptor {
         } ref;
 
         struct {
-            void (*content_free)(void *opaque, uint8_t *data);
-            int  (*content_clone)(AVBufferRef **ref, CodedBitstreamUnit *unit);
+            void (*content_free)(FFRefStructOpaque opaque, void *content);
+            int  (*content_clone)(void **new_content, CodedBitstreamUnit *unit);
         } complex;
     } type;
 } CodedBitstreamUnitTypeDescriptor;