obs-nvenc: Abort encoder init if custom options are invalid

plugins/obs-nvenc/data/locale/en-US.ini

@@ -53,6 +53,7 @@ SplitEncode.ThreeWay="Three-way split"
 
 Opts="Custom Encoder Options"
 Opts.TT="Space-separated list of options to apply to the rate control and codec settings,\nbased their names in the nvEncodeAPI header.\ne.g. \"lookaheadDepth=16 aqStrength=4\""
+Opts.Invalid="Invalid Custom Encoder options, check the log for details.<br><br>See our <a href=\"https://obsproject.com/kb/advanced-nvenc-options\">Knowledge Base Article</a> for information on available options."
 
 Error="Failed to open NVENC codec: %1"
 GenericError="Try installing the latest <a href=\"https://obsproject.com/go/nvidia-drivers\">NVIDIA driver</a> and closing other recording software that might be using NVENC such as NVIDIA ShadowPlay or Windows Game DVR."

plugins/obs-nvenc/nvenc-internal.h

@@ -202,5 +202,5 @@ obs_properties_t *hevc_nvenc_properties(void *);
 obs_properties_t *av1_nvenc_properties(void *);
 
 /* Custom argument parsing */
-void apply_user_args(struct nvenc_data *enc);
+bool apply_user_args(struct nvenc_data *enc);
 bool get_user_arg_int(struct nvenc_data *enc, const char *name, int *val);

plugins/obs-nvenc/nvenc-opts-parser.c

@@ -180,8 +180,10 @@ static bool apply_codec_opt(enum codec_type codec, struct obs_option *opt, NV_EN
 	return false;
 }
 
-void apply_user_args(struct nvenc_data *enc)
+bool apply_user_args(struct nvenc_data *enc)
 {
+	bool success = true;
+
 	for (size_t idx = 0; idx < enc->props.opts.count; idx++) {
 		struct obs_option *opt = &enc->props.opts.options[idx];
 
@@ -197,7 +199,10 @@ void apply_user_args(struct nvenc_data *enc)
 			continue;
 
 		warn("Unknown custom option: \"%s\"", opt->name);
+		success = false;
 	}
+
+	return success;
 }
 
 bool get_user_arg_int(struct nvenc_data *enc, const char *name, int *val)

plugins/obs-nvenc/nvenc.c

@@ -486,7 +486,10 @@ static bool init_encoder_h264(struct nvenc_data *enc, obs_data_t *settings)
 		config->profileGUID = NV_ENC_H264_PROFILE_HIGH_GUID;
 	}
 
-	apply_user_args(enc);
+	if (!apply_user_args(enc)) {
+		obs_encoder_set_last_error(enc->encoder, obs_module_text("Opts.Invalid"));
+		return false;
+	}
 
 	if (NV_FAILED(nv.nvEncInitializeEncoder(enc->session, &enc->params))) {
 		return false;
@@ -595,7 +598,10 @@ static bool init_encoder_hevc(struct nvenc_data *enc, obs_data_t *settings)
 	hevc_config->outputBitDepth = profile_is_10bpc ? NV_ENC_BIT_DEPTH_10 : NV_ENC_BIT_DEPTH_8;
 #endif
 
-	apply_user_args(enc);
+	if (!apply_user_args(enc)) {
+		obs_encoder_set_last_error(enc->encoder, obs_module_text("Opts.Invalid"));
+		return false;
+	}
 
 	if (NV_FAILED(nv.nvEncInitializeEncoder(enc->session, &enc->params))) {
 		return false;
@@ -679,7 +685,10 @@ static bool init_encoder_av1(struct nvenc_data *enc, obs_data_t *settings)
 	av1_config->numBwdRefs = 1;
 	av1_config->repeatSeqHdr = 1;
 
-	apply_user_args(enc);
+	if (!apply_user_args(enc)) {
+		obs_encoder_set_last_error(enc->encoder, obs_module_text("Opts.Invalid"));
+		return false;
+	}
 
 	if (NV_FAILED(nv.nvEncInitializeEncoder(enc->session, &enc->params))) {
 		return false;