Skip to content

Commit

Permalink
Merge pull request #450 from nyanmisaka/vpp-tm-cll
Browse files Browse the repository at this point in the history 
Fix some issues in QSV VPP filters
  • Loading branch information
@gnattu
gnattu committed Sep 6, 2024
2 parents ca08a83 + fb1447e commit af8156a
Show file tree
Hide file tree
Showing 2 changed files with 221 additions and 21 deletions.
140 changes: 135 additions & 5 deletions debian/patches/0019-backport-fixes-for-qsv-from-upstream.patch
View file
Original file line number Diff line number Diff line change
Expand Up @@ -936,7 +936,45 @@ Index: FFmpeg/libavfilter/qsvvpp.c
} else {
pix_fmt = link->format;
desc = av_pix_fmt_desc_get(pix_fmt);
@@ -603,6 +603,26 @@ static int init_vpp_session(AVFilterCont
@@ -440,11 +440,6 @@ static QSVFrame *submit_frame(QSVVPPCont
av_frame_free(&qsv_frame->frame);
return NULL;
}
-
- if (av_frame_copy_props(qsv_frame->frame, picref) < 0) {
- av_frame_free(&qsv_frame->frame);
- return NULL;
- }
} else
qsv_frame->frame = av_frame_clone(picref);

@@ -493,12 +488,6 @@ static QSVFrame *query_frame(QSVVPPConte
if (!out_frame->frame)
return NULL;

- ret = av_frame_copy_props(out_frame->frame, in);
- if (ret < 0) {
- av_log(ctx, AV_LOG_ERROR, "Failed to copy metadata fields from src to dst.\n");
- return NULL;
- }
-
ret = av_hwframe_get_buffer(outlink->hw_frames_ctx, out_frame->frame, 0);
if (ret < 0) {
av_log(ctx, AV_LOG_ERROR, "Can't allocate a surface.\n");
@@ -515,12 +504,6 @@ static QSVFrame *query_frame(QSVVPPConte
if (!out_frame->frame)
return NULL;

- ret = av_frame_copy_props(out_frame->frame, in);
- if (ret < 0) {
- av_log(ctx, AV_LOG_ERROR, "Failed to copy metadata fields from src to dst.\n");
- return NULL;
- }
-
ret = map_frame_to_surface(out_frame->frame,
&out_frame->surface);
if (ret < 0)
@@ -603,6 +586,26 @@ static int init_vpp_session(AVFilterCont
device_ctx = (AVHWDeviceContext *)device_ref->data;
device_hwctx = device_ctx->hwctx;

Expand All @@ -963,7 +1001,7 @@ Index: FFmpeg/libavfilter/qsvvpp.c
if (outlink->format == AV_PIX_FMT_QSV) {
AVHWFramesContext *out_frames_ctx;
AVBufferRef *out_frames_ref = av_hwframe_ctx_alloc(device_ref);
@@ -624,9 +644,15 @@ static int init_vpp_session(AVFilterCont
@@ -624,9 +627,15 @@ static int init_vpp_session(AVFilterCont
out_frames_ctx->width = FFALIGN(outlink->w, 32);
out_frames_ctx->height = FFALIGN(outlink->h, 32);
out_frames_ctx->sw_format = s->out_sw_format;
Expand All @@ -982,7 +1020,7 @@ Index: FFmpeg/libavfilter/qsvvpp.c
out_frames_hwctx->frame_type = s->out_mem_mode;

ret = av_hwframe_ctx_init(out_frames_ref);
@@ -652,26 +678,6 @@ static int init_vpp_session(AVFilterCont
@@ -652,26 +661,6 @@ static int init_vpp_session(AVFilterCont
} else
s->out_mem_mode = MFX_MEMTYPE_SYSTEM_MEMORY;

Expand All @@ -1009,15 +1047,33 @@ Index: FFmpeg/libavfilter/qsvvpp.c
ret = MFXVideoCORE_GetHandle(device_hwctx->session, handle_type, &handle);
if (ret < 0)
return ff_qsvvpp_print_error(avctx, ret, "Error getting the session handle");
@@ -1014,6 +1020,7 @@ int ff_qsvvpp_filter_frame(QSVVPPContext
@@ -957,7 +946,7 @@ int ff_qsvvpp_close(AVFilterContext *avc
return 0;
}

-int ff_qsvvpp_filter_frame(QSVVPPContext *s, AVFilterLink *inlink, AVFrame *picref)
+int ff_qsvvpp_filter_frame(QSVVPPContext *s, AVFilterLink *inlink, AVFrame *picref, AVFrame *propref)
{
AVFilterContext *ctx = inlink->dst;
AVFilterLink *outlink = ctx->outputs[0];
@@ -1014,6 +1003,16 @@ int ff_qsvvpp_filter_frame(QSVVPPContext
return AVERROR(EAGAIN);
break;
}
+
+ if (propref) {
+ ret1 = av_frame_copy_props(out_frame->frame, propref);
+ if (ret1 < 0) {
+ av_frame_free(&out_frame->frame);
+ av_log(ctx, AV_LOG_ERROR, "Failed to copy metadata fields from src to dst.\n");
+ return ret1;
+ }
+ }
+
out_frame->frame->pts = av_rescale_q(out_frame->surface.Data.TimeStamp,
default_tb, outlink->time_base);

@@ -1099,11 +1106,6 @@ int ff_qsvvpp_create_mfx_session(void *c
@@ -1099,11 +1098,6 @@ int ff_qsvvpp_create_mfx_session(void *c
if (sts < 0)
return ff_qsvvpp_print_error(ctx, sts,
"Error creating a MFX session");
Expand All @@ -1029,6 +1085,80 @@ Index: FFmpeg/libavfilter/qsvvpp.c

*psession = session;

Index: FFmpeg/libavfilter/qsvvpp.h
===================================================================
--- FFmpeg.orig/libavfilter/qsvvpp.h
+++ FFmpeg/libavfilter/qsvvpp.h
@@ -131,7 +131,7 @@ int ff_qsvvpp_init(AVFilterContext *avct
int ff_qsvvpp_close(AVFilterContext *avctx);

/* vpp filter frame and call the cb if needed */
-int ff_qsvvpp_filter_frame(QSVVPPContext *vpp, AVFilterLink *inlink, AVFrame *frame);
+int ff_qsvvpp_filter_frame(QSVVPPContext *vpp, AVFilterLink *inlink, AVFrame *frame, AVFrame *propref);

int ff_qsvvpp_print_iopattern(void *log_ctx, int mfx_iopattern,
const char *extra_string);
Index: FFmpeg/libavfilter/vf_overlay_qsv.c
===================================================================
--- FFmpeg.orig/libavfilter/vf_overlay_qsv.c
+++ FFmpeg/libavfilter/vf_overlay_qsv.c
@@ -230,13 +230,16 @@ static int process_frame(FFFrameSync *fs
{
AVFilterContext *ctx = fs->parent;
QSVVPPContext *qsv = fs->opaque;
- AVFrame *frame = NULL;
+ AVFrame *frame = NULL, *propref = NULL;
int ret = 0, i;

for (i = 0; i < ctx->nb_inputs; i++) {
ret = ff_framesync_get_frame(fs, i, &frame, 0);
- if (ret == 0)
- ret = ff_qsvvpp_filter_frame(qsv, ctx->inputs[i], frame);
+ if (ret == 0) {
+ if (i == 0)
+ propref = frame;
+ ret = ff_qsvvpp_filter_frame(qsv, ctx->inputs[i], frame, propref);
+ }
if (ret < 0 && ret != AVERROR(EAGAIN))
break;
}
Index: FFmpeg/libavfilter/vf_stack_qsv.c
===================================================================
--- FFmpeg.orig/libavfilter/vf_stack_qsv.c
+++ FFmpeg/libavfilter/vf_stack_qsv.c
@@ -70,13 +70,16 @@ static int process_frame(FFFrameSync *fs
{
AVFilterContext *ctx = fs->parent;
QSVVPPContext *qsv = fs->opaque;
- AVFrame *frame = NULL;
+ AVFrame *frame = NULL, *propref = NULL;
int ret = 0;

for (int i = 0; i < ctx->nb_inputs; i++) {
ret = ff_framesync_get_frame(fs, i, &frame, 0);
- if (ret == 0)
- ret = ff_qsvvpp_filter_frame(qsv, ctx->inputs[i], frame);
+ if (ret == 0) {
+ if (i == 0)
+ propref = frame;
+ ret = ff_qsvvpp_filter_frame(qsv, ctx->inputs[i], frame, propref);
+ }
if (ret < 0 && ret != AVERROR(EAGAIN))
break;
}
Index: FFmpeg/libavfilter/vf_vpp_qsv.c
===================================================================
--- FFmpeg.orig/libavfilter/vf_vpp_qsv.c
+++ FFmpeg/libavfilter/vf_vpp_qsv.c
@@ -748,7 +748,7 @@ static int activate(AVFilterContext *ctx

if (qsv->session) {
if (in || qsv->eof) {
- ret = ff_qsvvpp_filter_frame(qsv, inlink, in);
+ ret = ff_qsvvpp_filter_frame(qsv, inlink, in, in);
av_frame_free(&in);
if (ret == AVERROR(EAGAIN))
goto not_ready;
Index: FFmpeg/libavutil/hwcontext_qsv.c
===================================================================
--- FFmpeg.orig/libavutil/hwcontext_qsv.c
Expand Down
102 changes: 86 additions & 16 deletions debian/patches/0021-add-fixes-for-qsv-vpp-filters.patch
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ Index: FFmpeg/libavfilter/qsvvpp.c
return ret;
}

@@ -463,8 +463,12 @@ static QSVFrame *submit_frame(QSVVPPCont
@@ -458,8 +458,12 @@ static QSVFrame *submit_frame(QSVVPPCont
!(qsv_frame->frame->flags & AV_FRAME_FLAG_INTERLACED) ? MFX_PICSTRUCT_PROGRESSIVE :
((qsv_frame->frame->flags & AV_FRAME_FLAG_TOP_FIELD_FIRST) ? MFX_PICSTRUCT_FIELD_TFF :
MFX_PICSTRUCT_FIELD_BFF);
Expand All @@ -25,23 +25,66 @@ Index: FFmpeg/libavfilter/qsvvpp.c
else if (qsv_frame->frame->repeat_pict == 2)
qsv_frame->surface.Info.PicStruct |= MFX_PICSTRUCT_FRAME_DOUBLING;
else if (qsv_frame->frame->repeat_pict == 4)
@@ -986,6 +990,23 @@ int ff_qsvvpp_filter_frame(QSVVPPContext
return AVERROR(ENOMEM);
}

+ /* Copy metadata before initializing vpp session,
+ * which contains callback to drop the HDR metadata */
+ if (propref) {
+ int flags = out_frame->frame->flags;
+ int64_t duration = out_frame->frame->duration;
+
+ ret1 = av_frame_copy_props(out_frame->frame, propref);
+ if (ret1 < 0) {
+ av_frame_free(&out_frame->frame);
+ av_log(ctx, AV_LOG_ERROR, "Failed to copy metadata fields from src to dst.\n");
+ return ret1;
+ }
+
+ out_frame->frame->flags = flags;
+ out_frame->frame->duration = duration;
+ }
+
ret = qsvvpp_init_vpp_session(ctx, s, in_frame, out_frame);
if (ret)
return ret;
@@ -1004,15 +1025,6 @@ int ff_qsvvpp_filter_frame(QSVVPPContext
break;
}

- if (propref) {
- ret1 = av_frame_copy_props(out_frame->frame, propref);
- if (ret1 < 0) {
- av_frame_free(&out_frame->frame);
- av_log(ctx, AV_LOG_ERROR, "Failed to copy metadata fields from src to dst.\n");
- return ret1;
- }
- }
-
out_frame->frame->pts = av_rescale_q(out_frame->surface.Data.TimeStamp,
default_tb, outlink->time_base);

Index: FFmpeg/libavfilter/vf_overlay_qsv.c
===================================================================
--- FFmpeg.orig/libavfilter/vf_overlay_qsv.c
+++ FFmpeg/libavfilter/vf_overlay_qsv.c
@@ -228,40 +228,47 @@ static int config_overlay_input(AVFilter
@@ -228,43 +228,47 @@ static int config_overlay_input(AVFilter

static int process_frame(FFFrameSync *fs)
{
- AVFilterContext *ctx = fs->parent;
- QSVVPPContext *qsv = fs->opaque;
- AVFrame *frame = NULL;
- AVFrame *frame = NULL, *propref = NULL;
- int ret = 0, i;
-
- for (i = 0; i < ctx->nb_inputs; i++) {
- ret = ff_framesync_get_frame(fs, i, &frame, 0);
- if (ret == 0)
- ret = ff_qsvvpp_filter_frame(qsv, ctx->inputs[i], frame);
- if (ret == 0) {
- if (i == 0)
- propref = frame;
- ret = ff_qsvvpp_filter_frame(qsv, ctx->inputs[i], frame, propref);
- }
- if (ret < 0 && ret != AVERROR(EAGAIN))
- break;
- }
Expand All @@ -65,13 +108,13 @@ Index: FFmpeg/libavfilter/vf_overlay_qsv.c

- return ret;
+ /* composite main frame */
+ ret = ff_qsvvpp_filter_frame(qsv, in0, main);
+ ret = ff_qsvvpp_filter_frame(qsv, in0, main, main);
+ if (ret < 0 && ret != AVERROR(EAGAIN))
+ return ret;
+
+ /* composite overlay frame */
+ /* or overwrite main frame again if the overlay frame isn't ready yet */
+ return ff_qsvvpp_filter_frame(qsv, overlay ? in1 : in0, overlay ? overlay : main);
+ return ff_qsvvpp_filter_frame(qsv, overlay ? in1 : in0, overlay ? overlay : main, main);
}

static int init_framesync(AVFilterContext *ctx)
Expand Down Expand Up @@ -102,7 +145,7 @@ Index: FFmpeg/libavfilter/vf_overlay_qsv.c

return ff_framesync_configure(&s->fs);
}
@@ -282,12 +289,6 @@ static int config_output(AVFilterLink *o
@@ -285,12 +289,6 @@ static int config_output(AVFilterLink *o
return AVERROR(EINVAL);
} else if (in0->format == AV_PIX_FMT_QSV) {
AVHWFramesContext *hw_frame0 = (AVHWFramesContext *)in0->hw_frames_ctx->data;
Expand All @@ -115,15 +158,15 @@ Index: FFmpeg/libavfilter/vf_overlay_qsv.c
vpp->qsv_param.out_sw_format = hw_frame0->sw_format;
}

@@ -369,6 +370,7 @@ static int overlay_qsv_query_formats(AVF
@@ -372,6 +370,7 @@ static int overlay_qsv_query_formats(AVF
static const enum AVPixelFormat main_in_fmts[] = {
AV_PIX_FMT_YUV420P,
AV_PIX_FMT_NV12,
+ AV_PIX_FMT_P010,
AV_PIX_FMT_YUYV422,
AV_PIX_FMT_RGB32,
AV_PIX_FMT_QSV,
@@ -376,6 +378,7 @@ static int overlay_qsv_query_formats(AVF
@@ -379,6 +378,7 @@ static int overlay_qsv_query_formats(AVF
};
static const enum AVPixelFormat out_pix_fmts[] = {
AV_PIX_FMT_NV12,
Expand Down Expand Up @@ -166,7 +209,34 @@ Index: FFmpeg/libavfilter/vf_vpp_qsv.c
static int vpp_set_frame_ext_params(AVFilterContext *ctx, const AVFrame *in, AVFrame *out, QSVVPPFrameParam *fp)
{
#if QSV_ONEVPL
@@ -494,9 +518,9 @@ static int vpp_set_frame_ext_params(AVFi
@@ -461,14 +485,19 @@ static int vpp_set_frame_ext_params(AVFi

memset(&clli_conf, 0, sizeof(mfxExtContentLightLevelInfo));
sd = av_frame_get_side_data(in, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL);
- if (vpp->tonemap && sd) {
- AVContentLightMetadata *clm = (AVContentLightMetadata *)sd->data;
+ if (vpp->tonemap) {
+ AVContentLightMetadata *clm = sd ? (AVContentLightMetadata *)sd->data : NULL;

- clli_conf.Header.BufferId = MFX_EXTBUFF_CONTENT_LIGHT_LEVEL_INFO;
- clli_conf.Header.BufferSz = sizeof(mfxExtContentLightLevelInfo);
- clli_conf.MaxContentLightLevel = FFMIN(clm->MaxCLL, 65535);
- clli_conf.MaxPicAverageLightLevel = FFMIN(clm->MaxFALL, 65535);
- tm = 1;
+ // Dumped from VP HAL, VPL requires at least one type of the metadata to trigger tone-mapping
+ #define HAL_HDR_DEFAULT_MAXCLL 4000
+ #define HAL_HDR_DEFAULT_MAXFALL 400
+ if (clm || !tm) {
+ clli_conf.Header.BufferId = MFX_EXTBUFF_CONTENT_LIGHT_LEVEL_INFO;
+ clli_conf.Header.BufferSz = sizeof(mfxExtContentLightLevelInfo);
+ clli_conf.MaxContentLightLevel = FFMIN(clm ? clm->MaxCLL : HAL_HDR_DEFAULT_MAXCLL, 65535);
+ clli_conf.MaxPicAverageLightLevel = FFMIN(clm ? clm->MaxFALL : HAL_HDR_DEFAULT_MAXFALL, 65535);
+ tm = 1;
+ }
}

if (tm) {
@@ -494,9 +523,9 @@ static int vpp_set_frame_ext_params(AVFi
outvsi_conf.Header.BufferId = MFX_EXTBUFF_VIDEO_SIGNAL_INFO_OUT;
outvsi_conf.Header.BufferSz = sizeof(mfxExtVideoSignalInfo);
outvsi_conf.VideoFullRange = (out->color_range == AVCOL_RANGE_JPEG);
Expand All @@ -179,7 +249,7 @@ Index: FFmpeg/libavfilter/vf_vpp_qsv.c
outvsi_conf.ColourDescriptionPresent = 1;

if (memcmp(&vpp->invsi_conf, &invsi_conf, sizeof(mfxExtVideoSignalInfo)) ||
@@ -686,12 +710,24 @@ static int config_output(AVFilterLink *o
@@ -686,12 +715,24 @@ static int config_output(AVFilterLink *o

if (inlink->w != outlink->w || inlink->h != outlink->h || in_format != vpp->out_format) {
if (QSV_RUNTIME_VERSION_ATLEAST(mfx_version, 1, 19)) {
Expand Down Expand Up @@ -208,7 +278,7 @@ Index: FFmpeg/libavfilter/vf_vpp_qsv.c

INIT_MFX_EXTBUF(scale_conf, MFX_EXTBUFF_VPP_SCALING);
SET_MFX_PARAM_FIELD(scale_conf, ScalingMode, mode);
@@ -880,19 +916,13 @@ static const AVOption vpp_options[] = {
@@ -880,19 +921,13 @@ static const AVOption vpp_options[] = {
{ "height", "Output video height(0=input video height, -1=keep input video aspect)", OFFSET(oh), AV_OPT_TYPE_STRING, { .str="w*ch/cw" }, 0, 255, .flags = FLAGS },
{ "format", "Output pixel format", OFFSET(output_format_str), AV_OPT_TYPE_STRING, { .str = "same" }, .flags = FLAGS },
{ "async_depth", "Internal parallelization depth, the higher the value the higher the latency.", OFFSET(qsv.async_depth), AV_OPT_TYPE_INT, { .i64 = 4 }, 0, INT_MAX, .flags = FLAGS },
Expand All @@ -229,7 +299,7 @@ Index: FFmpeg/libavfilter/vf_vpp_qsv.c

{ "rate", "Generate output at frame rate or field rate, available only for deinterlace mode",
OFFSET(field_rate), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, FLAGS, .unit = "rate" },
@@ -923,8 +953,9 @@ static const AVOption vpp_options[] = {
@@ -923,8 +958,9 @@ static const AVOption vpp_options[] = {
{ "out_color_transfer", "Output color transfer characteristics",
OFFSET(color_transfer_str), AV_OPT_TYPE_STRING, { .str = NULL }, .flags = FLAGS },

Expand All @@ -240,7 +310,7 @@ Index: FFmpeg/libavfilter/vf_vpp_qsv.c
{ NULL }
};

@@ -978,19 +1009,14 @@ static const AVOption qsvscale_options[]
@@ -978,19 +1014,14 @@ static const AVOption qsvscale_options[]
{ "h", "Output video height(0=input video height, -1=keep input video aspect)", OFFSET(oh), AV_OPT_TYPE_STRING, { .str = "ih" }, .flags = FLAGS },
{ "format", "Output pixel format", OFFSET(output_format_str), AV_OPT_TYPE_STRING, { .str = "same" }, .flags = FLAGS },

Expand All @@ -262,7 +332,7 @@ Index: FFmpeg/libavfilter/vf_vpp_qsv.c
{ NULL },
};

@@ -1015,6 +1041,7 @@ static const AVOption qsvdeint_options[]
@@ -1015,6 +1046,7 @@ static const AVOption qsvdeint_options[]
{ "bob", "bob algorithm", 0, AV_OPT_TYPE_CONST, {.i64 = MFX_DEINTERLACING_BOB}, MFX_DEINTERLACING_BOB, MFX_DEINTERLACING_ADVANCED, FLAGS, .unit = "mode"},
{ "advanced", "Motion adaptive algorithm", 0, AV_OPT_TYPE_CONST, {.i64 = MFX_DEINTERLACING_ADVANCED}, MFX_DEINTERLACING_BOB, MFX_DEINTERLACING_ADVANCED, FLAGS, .unit = "mode"},

Expand Down

0 comments on commit af8156a

Please sign in to comment.