From 699e89d8596e8f96c3d7d4448998ca73d2efac87 Mon Sep 17 00:00:00 2001 From: jpark37 Date: Fri, 27 May 2022 23:06:05 -0700 Subject: [PATCH] obs-ffmpeg: Improve dialog text for NVENC errors Add dialog messages for lack of 10-bit support, and exceeding the hardware B-frame limit. Also move code around to test these just once. --- plugins/obs-ffmpeg/jim-nvenc-helpers.c | 20 +++++ plugins/obs-ffmpeg/jim-nvenc.c | 104 ++++++++++++++++--------- plugins/obs-ffmpeg/jim-nvenc.h | 1 + 3 files changed, 87 insertions(+), 38 deletions(-) diff --git a/plugins/obs-ffmpeg/jim-nvenc-helpers.c b/plugins/obs-ffmpeg/jim-nvenc-helpers.c index af3c6f386..49954e465 100644 --- a/plugins/obs-ffmpeg/jim-nvenc-helpers.c +++ b/plugins/obs-ffmpeg/jim-nvenc-helpers.c @@ -10,6 +10,26 @@ NV_CREATE_INSTANCE_FUNC nv_create_instance = NULL; #define error(format, ...) blog(LOG_ERROR, "[jim-nvenc] " format, ##__VA_ARGS__) +bool nv_fail(obs_encoder_t *encoder, const char *format, ...) +{ + struct dstr message = {0}; + struct dstr error_message = {0}; + + va_list args; + va_start(args, format); + dstr_vprintf(&message, format, args); + va_end(args); + + dstr_printf(&error_message, "NVENC Error: %s", message.array); + obs_encoder_set_last_error(encoder, error_message.array); + error("%s", error_message.array); + + dstr_free(&error_message); + dstr_free(&message); + + return true; +} + bool nv_failed(obs_encoder_t *encoder, NVENCSTATUS err, const char *func, const char *call) { diff --git a/plugins/obs-ffmpeg/jim-nvenc.c b/plugins/obs-ffmpeg/jim-nvenc.c index ee7b5e466..45b7ac3fe 100644 --- a/plugins/obs-ffmpeg/jim-nvenc.c +++ b/plugins/obs-ffmpeg/jim-nvenc.c @@ -89,6 +89,7 @@ struct nv_bitstream { HANDLE event; }; +#define NV_FAIL(format, ...) nv_fail(enc->encoder, format, __VA_ARGS__) #define NV_FAILED(x) nv_failed(enc->encoder, x, __FUNCTION__, #x) static bool nv_bitstream_init(struct nvenc_data *enc, struct nv_bitstream *bs) @@ -353,7 +354,7 @@ static bool init_session(struct nvenc_data *enc) } static bool init_encoder_h264(struct nvenc_data *enc, obs_data_t *settings, - bool psycho_aq) + int bf, bool psycho_aq) { const char *rc = obs_data_get_string(settings, "rate_control"); int bitrate = (int)obs_data_get_int(settings, "bitrate"); @@ -363,17 +364,9 @@ static bool init_encoder_h264(struct nvenc_data *enc, obs_data_t *settings, const char *preset = obs_data_get_string(settings, "preset"); const char *profile = obs_data_get_string(settings, "profile"); bool lookahead = obs_data_get_bool(settings, "lookahead"); - int bf = (int)obs_data_get_int(settings, "bf"); bool vbr = astrcmpi(rc, "VBR") == 0; NVENCSTATUS err; - const int bf_max = nv_get_cap_h264(enc, NV_ENC_CAPS_NUM_MAX_BFRAMES); - if (bf > bf_max) { - error("Max B-frames setting (%d) is more than GPU supports (%d)", - bf, bf_max); - return false; - } - video_t *video = obs_encoder_video(enc->encoder); const struct video_output_info *voi = video_output_get_info(video); @@ -633,7 +626,7 @@ static bool init_encoder_h264(struct nvenc_data *enc, obs_data_t *settings, #ifdef ENABLE_HEVC static bool init_encoder_hevc(struct nvenc_data *enc, obs_data_t *settings, - bool psycho_aq) + int bf, bool psycho_aq) { const char *rc = obs_data_get_string(settings, "rate_control"); int bitrate = (int)obs_data_get_int(settings, "bitrate"); @@ -643,17 +636,9 @@ static bool init_encoder_hevc(struct nvenc_data *enc, obs_data_t *settings, const char *preset = obs_data_get_string(settings, "preset"); const char *profile = obs_data_get_string(settings, "profile"); bool lookahead = obs_data_get_bool(settings, "lookahead"); - int bf = (int)obs_data_get_int(settings, "bf"); bool vbr = astrcmpi(rc, "VBR") == 0; NVENCSTATUS err; - const int bf_max = nv_get_cap_hevc(enc, NV_ENC_CAPS_NUM_MAX_BFRAMES); - if (bf > bf_max) { - error("Max B-frames setting (%d) is more than GPU supports (%d)", - bf, bf_max); - return false; - } - video_t *video = obs_encoder_video(enc->encoder); const struct video_output_info *voi = video_output_get_info(video); @@ -954,8 +939,67 @@ static bool init_textures(struct nvenc_data *enc) static void nvenc_destroy(void *data); +static bool init_specific_encoder(struct nvenc_data *enc, bool hevc, + obs_data_t *settings, obs_encoder_t *encoder, + int bf, bool psycho_aq) +{ + bool init = false; + +#ifdef ENABLE_HEVC + if (hevc) + return init_encoder_hevc(enc, settings, bf, psycho_aq); +#endif + + return init_encoder_h264(enc, settings, bf, psycho_aq); +} + +static bool init_encoder(struct nvenc_data *enc, bool hevc, + obs_data_t *settings, obs_encoder_t *encoder) +{ + const int bf = (int)obs_data_get_int(settings, "bf"); + const bool psycho_aq = obs_data_get_bool(settings, "psycho_aq"); +#ifdef ENABLE_HEVC + const bool support_10bit = + hevc ? nv_get_cap_hevc(enc, NV_ENC_CAPS_SUPPORT_10BIT_ENCODE) + : nv_get_cap_h264(enc, NV_ENC_CAPS_SUPPORT_10BIT_ENCODE); + const int bf_max = + hevc ? nv_get_cap_hevc(enc, NV_ENC_CAPS_NUM_MAX_BFRAMES) + : nv_get_cap_h264(enc, NV_ENC_CAPS_NUM_MAX_BFRAMES); +#else + const bool support_10bit = + nv_get_cap_h264(enc, NV_ENC_CAPS_SUPPORT_10BIT_ENCODE); + const int bf_max = nv_get_cap_h264(enc, NV_ENC_CAPS_NUM_MAX_BFRAMES); +#endif + + if (obs_p010_tex_active() && !support_10bit) { + NV_FAIL("Cannot perform 10-bit encode on this encoder"); + return false; + } + + if (bf > bf_max) { + NV_FAIL("Max B-frames setting (%d) is more than encoder supports (%d)", + bf, bf_max); + return false; + } + + if (!init_specific_encoder(enc, hevc, settings, encoder, bf, + psycho_aq)) { + if (!psycho_aq) + return false; + + blog(LOG_WARNING, "[jim-nvenc] init_specific_encoder failed, " + "trying again without Psycho Visual Tuning"); + if (!init_specific_encoder(enc, hevc, settings, encoder, bf, + psycho_aq)) { + return false; + } + } + + return true; +} + static void *nvenc_create_internal(bool hevc, obs_data_t *settings, - obs_encoder_t *encoder, bool psycho_aq) + obs_encoder_t *encoder) { NV_ENCODE_API_FUNCTION_LIST init = {NV_ENCODE_API_FUNCTION_LIST_VER}; struct nvenc_data *enc = bzalloc(sizeof(*enc)); @@ -974,17 +1018,8 @@ static void *nvenc_create_internal(bool hevc, obs_data_t *settings, if (!init_session(enc)) { goto fail; } -#ifdef ENABLE_HEVC - if (hevc) { - if (!init_encoder_hevc(enc, settings, psycho_aq)) { - goto fail; - } - } else -#endif - { - if (!init_encoder_h264(enc, settings, psycho_aq)) { - goto fail; - } + if (!init_encoder(enc, hevc, settings, encoder)) { + goto fail; } if (!init_bitstreams(enc)) { goto fail; @@ -1027,14 +1062,7 @@ static void *nvenc_create_h264_hevc(bool hevc, obs_data_t *settings, goto reroute; } - const bool psycho_aq = obs_data_get_bool(settings, "psycho_aq"); - struct nvenc_data *enc = - nvenc_create_internal(hevc, settings, encoder, psycho_aq); - if ((enc == NULL) && psycho_aq) { - blog(LOG_WARNING, "[jim-nvenc] nvenc_create_internal failed, " - "trying again without Psycho Visual Tuning"); - enc = nvenc_create_internal(hevc, settings, encoder, false); - } + struct nvenc_data *enc = nvenc_create_internal(hevc, settings, encoder); if (enc) { return enc; diff --git a/plugins/obs-ffmpeg/jim-nvenc.h b/plugins/obs-ffmpeg/jim-nvenc.h index cdd60aa84..66290b64b 100644 --- a/plugins/obs-ffmpeg/jim-nvenc.h +++ b/plugins/obs-ffmpeg/jim-nvenc.h @@ -13,5 +13,6 @@ extern const char *nv_error_name(NVENCSTATUS err); extern NV_ENCODE_API_FUNCTION_LIST nv; extern NV_CREATE_INSTANCE_FUNC nv_create_instance; extern bool init_nvenc(obs_encoder_t *encoder); +bool nv_fail(obs_encoder_t *encoder, const char *format, ...); bool nv_failed(obs_encoder_t *encoder, NVENCSTATUS err, const char *func, const char *call);