diff --git a/libavutil/frame.c b/libavutil/frame.c
index 53e6174223..9050946c48 100644
--- a/libavutil/frame.c
+++ b/libavutil/frame.c
@@ -349,6 +349,7 @@ FF_ENABLE_DEPRECATION_WARNINGS
                 wipe_side_data(dst);
                 return AVERROR(ENOMEM);
             }
+            if (sd_src->buf) {
             sd_dst->buf = av_buffer_ref(sd_src->buf);
             if (!sd_dst->buf) {
                 wipe_side_data(dst);
@@ -356,6 +357,7 @@ FF_ENABLE_DEPRECATION_WARNINGS
             }
             sd_dst->data = sd_dst->buf->data;
             sd_dst->size = sd_dst->buf->size;
+            }
         }
         av_dict_copy(&sd_dst->metadata, sd_src->metadata, 0);
     }