diff --git a/libavcodec/av1dec.c b/libavcodec/av1dec.c
index cc178464b9..09339529f7 100644
--- a/libavcodec/av1dec.c
+++ b/libavcodec/av1dec.c
@@ -907,17 +907,11 @@ static int av1_frame_alloc(AVCodecContext *avctx, AV1Frame *f)
         break;
     }
 
-    if (avctx->hwaccel) {
-        const AVHWAccel *hwaccel = avctx->hwaccel;
-        if (hwaccel->frame_priv_data_size) {
-            f->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(avctx, hwaccel);
-            if (!f->hwaccel_priv_buf) {
-                ret = AVERROR(ENOMEM);
-                goto fail;
-            }
-            f->hwaccel_picture_private = f->hwaccel_priv_buf->data;
-        }
-    }
+    ret = ff_hwaccel_frame_priv_alloc(avctx, &f->hwaccel_picture_private,
+                                      &f->hwaccel_priv_buf);
+    if (ret < 0)
+        goto fail;
+
     return 0;
 
 fail:
diff --git a/libavcodec/decode.c b/libavcodec/decode.c
index 53c90cddb7..be761873a3 100644
--- a/libavcodec/decode.c
+++ b/libavcodec/decode.c
@@ -1788,24 +1788,36 @@ int ff_copy_palette(void *dst, const AVPacket *src, void *logctx)
     return 0;
 }
 
-AVBufferRef *ff_hwaccel_frame_priv_alloc(AVCodecContext *avctx,
-                                         const AVHWAccel *hwaccel)
+int ff_hwaccel_frame_priv_alloc(AVCodecContext *avctx, void **hwaccel_picture_private,
+                                AVBufferRef **hwaccel_priv_buf)
 {
+    const AVHWAccel *hwaccel = avctx->hwaccel;
     AVBufferRef *ref;
-    AVHWFramesContext *frames_ctx = (AVHWFramesContext *)avctx->hw_frames_ctx->data;
-    uint8_t *data = av_mallocz(hwaccel->frame_priv_data_size);
+    AVHWFramesContext *frames_ctx;
+    uint8_t *data;
+
+    if (!hwaccel || !hwaccel->frame_priv_data_size)
+        return 0;
+
+    av_assert0(!*hwaccel_picture_private);
+    data       = av_mallocz(hwaccel->frame_priv_data_size);
     if (!data)
-        return NULL;
+        return AVERROR(ENOMEM);
+
+    frames_ctx = (AVHWFramesContext *)avctx->hw_frames_ctx->data;
 
     ref = av_buffer_create(data, hwaccel->frame_priv_data_size,
                            hwaccel->free_frame_priv,
                            frames_ctx->device_ctx, 0);
     if (!ref) {
         av_free(data);
-        return NULL;
+        return AVERROR(ENOMEM);
     }
 
-    return ref;
+    *hwaccel_priv_buf        = ref;
+    *hwaccel_picture_private = ref->data;
+
+    return 0;
 }
 
 void ff_decode_flush_buffers(AVCodecContext *avctx)
diff --git a/libavcodec/decode.h b/libavcodec/decode.h
index 2b9fe59907..a52152e4a7 100644
--- a/libavcodec/decode.h
+++ b/libavcodec/decode.h
@@ -140,14 +140,17 @@ int ff_side_data_update_matrix_encoding(AVFrame *frame,
                                         enum AVMatrixEncoding matrix_encoding);
 
 /**
- * Allocate a hwaccel frame private data and create an AVBufferRef
- * from it.
+ * Allocate a hwaccel frame private data if the provided avctx
+ * uses a hwaccel method that needs it. The private data will
+ * be refcounted via the AVBuffer API (if allocated).
  *
- * @param     avctx   The codec context which to attach as an opaque value
- * @param     hwaccel The hwaccel for which to allocate
- * @return            The allocated buffer
+ * @param  avctx                   The codec context
+ * @param  hwaccel_picture_private Pointer to return hwaccel_picture_private
+ * @param  hwaccel_priv_buf        Pointer to return the AVBufferRef owning
+ *                                 hwaccel_picture_private
+ * @return 0 on success, < 0 on error
  */
-AVBufferRef *ff_hwaccel_frame_priv_alloc(AVCodecContext *avctx,
-                                         const AVHWAccel *hwaccel);
+int ff_hwaccel_frame_priv_alloc(AVCodecContext *avctx, void **hwaccel_picture_private,
+                                AVBufferRef **hwaccel_priv_buf);
 
 #endif /* AVCODEC_DECODE_H */
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index 8baf9944c5..6cd7bb8fe7 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -205,16 +205,11 @@ static int alloc_picture(H264Context *h, H264Picture *pic)
             goto fail;
     }
 
-    if (h->avctx->hwaccel) {
-        const AVHWAccel *hwaccel = h->avctx->hwaccel;
-        av_assert0(!pic->hwaccel_picture_private);
-        if (hwaccel->frame_priv_data_size) {
-            pic->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(h->avctx, hwaccel);
-            if (!pic->hwaccel_priv_buf)
-                return AVERROR(ENOMEM);
-            pic->hwaccel_picture_private = pic->hwaccel_priv_buf->data;
-        }
-    }
+    ret = ff_hwaccel_frame_priv_alloc(h->avctx, &pic->hwaccel_picture_private,
+                                      &pic->hwaccel_priv_buf);
+    if (ret < 0)
+        goto fail;
+
     if (CONFIG_GRAY && !h->avctx->hwaccel && h->flags & AV_CODEC_FLAG_GRAY && pic->f->data[2]) {
         int h_chroma_shift, v_chroma_shift;
         av_pix_fmt_get_chroma_sub_sample(pic->f->format,
diff --git a/libavcodec/hevc_refs.c b/libavcodec/hevc_refs.c
index 3aa8e19950..c5c1203ef8 100644
--- a/libavcodec/hevc_refs.c
+++ b/libavcodec/hevc_refs.c
@@ -118,16 +118,10 @@ static HEVCFrame *alloc_frame(HEVCContext *s)
             (s->sei.picture_timing.picture_struct == AV_PICTURE_STRUCTURE_BOTTOM_FIELD))
             frame->frame->flags |= AV_FRAME_FLAG_INTERLACED;
 
-        if (s->avctx->hwaccel) {
-            const AVHWAccel *hwaccel = s->avctx->hwaccel;
-            av_assert0(!frame->hwaccel_picture_private);
-            if (hwaccel->frame_priv_data_size) {
-                frame->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(s->avctx, hwaccel);
-                if (!frame->hwaccel_priv_buf)
-                    goto fail;
-                frame->hwaccel_picture_private = frame->hwaccel_priv_buf->data;
-            }
-        }
+        ret = ff_hwaccel_frame_priv_alloc(s->avctx, &frame->hwaccel_picture_private,
+                                          &frame->hwaccel_priv_buf);
+        if (ret < 0)
+            goto fail;
 
         return frame;
 fail:
diff --git a/libavcodec/mpegpicture.c b/libavcodec/mpegpicture.c
index 71c7a3fd70..b7c804c8ec 100644
--- a/libavcodec/mpegpicture.c
+++ b/libavcodec/mpegpicture.c
@@ -171,17 +171,10 @@ static int alloc_frame_buffer(AVCodecContext *avctx,  Picture *pic,
         pic->f->height = avctx->height;
     }
 
-    if (avctx->hwaccel) {
-        assert(!pic->hwaccel_picture_private);
-        if (avctx->hwaccel->frame_priv_data_size) {
-            pic->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(avctx, avctx->hwaccel);
-            if (!pic->hwaccel_priv_buf) {
-                av_log(avctx, AV_LOG_ERROR, "alloc_frame_buffer() failed (hwaccel private data allocation)\n");
-                return -1;
-            }
-            pic->hwaccel_picture_private = pic->hwaccel_priv_buf->data;
-        }
-    }
+    ret = ff_hwaccel_frame_priv_alloc(avctx, &pic->hwaccel_picture_private,
+                                      &pic->hwaccel_priv_buf);
+    if (ret < 0)
+        return ret;
 
     if ((linesize   &&   linesize != pic->f->linesize[0]) ||
         (uvlinesize && uvlinesize != pic->f->linesize[1])) {
diff --git a/libavcodec/vp8.c b/libavcodec/vp8.c
index 50afe19b7a..15a2961f56 100644
--- a/libavcodec/vp8.c
+++ b/libavcodec/vp8.c
@@ -104,23 +104,21 @@ static int vp8_alloc_frame(VP8Context *s, VP8Frame *f, int ref)
     if ((ret = ff_thread_get_ext_buffer(s->avctx, &f->tf,
                                         ref ? AV_GET_BUFFER_FLAG_REF : 0)) < 0)
         return ret;
-    if (!(f->seg_map = av_buffer_allocz(s->mb_width * s->mb_height)))
+    if (!(f->seg_map = av_buffer_allocz(s->mb_width * s->mb_height))) {
+        ret = AVERROR(ENOMEM);
         goto fail;
-    if (s->avctx->hwaccel) {
-        const AVHWAccel *hwaccel = s->avctx->hwaccel;
-        if (hwaccel->frame_priv_data_size) {
-            f->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(s->avctx, hwaccel);
-            if (!f->hwaccel_priv_buf)
-                goto fail;
-            f->hwaccel_picture_private = f->hwaccel_priv_buf->data;
-        }
     }
+    ret = ff_hwaccel_frame_priv_alloc(s->avctx, &f->hwaccel_picture_private,
+                                      &f->hwaccel_priv_buf);
+    if (ret < 0)
+        goto fail;
+
     return 0;
 
 fail:
     av_buffer_unref(&f->seg_map);
     ff_thread_release_ext_buffer(s->avctx, &f->tf);
-    return AVERROR(ENOMEM);
+    return ret;
 }
 
 static void vp8_release_frame(VP8Context *s, VP8Frame *f)
diff --git a/libavcodec/vp9.c b/libavcodec/vp9.c
index 4f704ec0dd..751dc7271f 100644
--- a/libavcodec/vp9.c
+++ b/libavcodec/vp9.c
@@ -119,12 +119,14 @@ static int vp9_frame_alloc(AVCodecContext *avctx, VP9Frame *f)
         s->frame_extradata_pool = av_buffer_pool_init(sz * (1 + sizeof(VP9mvrefPair)), NULL);
         if (!s->frame_extradata_pool) {
             s->frame_extradata_pool_size = 0;
+            ret = AVERROR(ENOMEM);
             goto fail;
         }
         s->frame_extradata_pool_size = sz;
     }
     f->extradata = av_buffer_pool_get(s->frame_extradata_pool);
     if (!f->extradata) {
+        ret = AVERROR(ENOMEM);
         goto fail;
     }
     memset(f->extradata->data, 0, f->extradata->size);
@@ -132,22 +134,16 @@ static int vp9_frame_alloc(AVCodecContext *avctx, VP9Frame *f)
     f->segmentation_map = f->extradata->data;
     f->mv = (VP9mvrefPair *) (f->extradata->data + sz);
 
-    if (avctx->hwaccel) {
-        const AVHWAccel *hwaccel = avctx->hwaccel;
-        av_assert0(!f->hwaccel_picture_private);
-        if (hwaccel->frame_priv_data_size) {
-            f->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(avctx, hwaccel);
-            if (!f->hwaccel_priv_buf)
-                goto fail;
-            f->hwaccel_picture_private = f->hwaccel_priv_buf->data;
-        }
-    }
+    ret = ff_hwaccel_frame_priv_alloc(avctx, &f->hwaccel_picture_private,
+                                      &f->hwaccel_priv_buf);
+    if (ret < 0)
+        goto fail;
 
     return 0;
 
 fail:
     vp9_frame_unref(avctx, f);
-    return AVERROR(ENOMEM);
+    return ret;
 }
 
 static int vp9_frame_ref(AVCodecContext *avctx, VP9Frame *dst, VP9Frame *src)