From 64c353c0197af7bbea1f65380d2f82545b0bebe9 Mon Sep 17 00:00:00 2001 From: Richard Stanway Date: Fri, 23 Apr 2021 17:38:31 +0200 Subject: [PATCH] obs-filters: Initialize NVIDIA AFX in a thread Initializing NVAFX is slow as it has to load large models and can cause significant impact to OBS' startup time. This moves NVAFX initialization into a separate thread. NVAFX does not appear to be reentrant across threads, hence a global mutex is also used to avoid reentrancy issues. The downside of this change is that the first few seconds of audio after adding the filter or starting OBS will not be filtered, but this is unlikely to cause any real-world issues. --- plugins/obs-filters/noise-suppress-filter.c | 109 +++++++++++++------- 1 file changed, 72 insertions(+), 37 deletions(-) diff --git a/plugins/obs-filters/noise-suppress-filter.c b/plugins/obs-filters/noise-suppress-filter.c index 25eafc1d3..3a095a8fd 100644 --- a/plugins/obs-filters/noise-suppress-filter.c +++ b/plugins/obs-filters/noise-suppress-filter.c @@ -19,6 +19,7 @@ bool nvafx_loaded = false; #ifdef LIBNVAFX_ENABLED #include "nvafx-load.h" +#include #endif /* -------------------------------------------------------- */ @@ -114,6 +115,11 @@ struct noise_suppress_data { /* Resampler */ audio_resampler_t *nvafx_resampler; audio_resampler_t *nvafx_resampler_back; + + /* Initialization */ + bool nvafx_loading; + pthread_t nvafx_thread; + pthread_mutex_t nvafx_mutex; #endif /* PCM buffers */ float *copy_buffers[MAX_PREPROC_CHANNELS]; @@ -132,6 +138,11 @@ struct noise_suppress_data { DARRAY(float) output_data; }; +#ifdef LIBNVAFX_ENABLED +/* global mutex for nvafx load functions since they aren't thread-safe */ +pthread_mutex_t nvafx_initializer_mutex; +#endif + /* -------------------------------------------------------- */ #define SUP_MIN -60 @@ -152,6 +163,10 @@ static void noise_suppress_destroy(void *data) { struct noise_suppress_data *ng = data; +#ifdef LIBNVAFX_ENABLED + pthread_mutex_lock(&ng->nvafx_mutex); +#endif + for (size_t i = 0; i < ng->channels; i++) { #ifdef LIBSPEEXDSP_ENABLED speex_preprocess_state_destroy(ng->spx_states[i]); @@ -190,6 +205,10 @@ static void noise_suppress_destroy(void *data) audio_resampler_destroy(ng->nvafx_resampler_back); } bfree(ng->model); + if (ng->use_nvafx) + pthread_join(ng->nvafx_thread, NULL); + pthread_mutex_unlock(&ng->nvafx_mutex); + pthread_mutex_destroy(&ng->nvafx_mutex); #endif bfree(ng->copy_buffers[0]); @@ -198,13 +217,17 @@ static void noise_suppress_destroy(void *data) bfree(ng); } -static inline bool nvafx_initialize(void *data) +static void *nvafx_initialize(void *data) { #ifdef LIBNVAFX_ENABLED struct noise_suppress_data *ng = data; int err; + if (!ng->use_nvafx || !nvafx_loaded) - return false; + return NULL; + + pthread_mutex_lock(&ng->nvafx_mutex); + pthread_mutex_lock(&nvafx_initializer_mutex); if (!ng->handle[0]) { ng->sample_rate = NVAFX_SAMPLE_RATE; @@ -215,8 +238,7 @@ static inline bool nvafx_initialize(void *data) do_log(LOG_ERROR, "NvAFX_CreateEffect() failed, error %i", err); - ng->use_nvafx = false; - return false; + goto failure; } err = NvAFX_SetU32(ng->handle[i], NVAFX_PARAM_DENOISER_SAMPLE_RATE, @@ -225,8 +247,7 @@ static inline bool nvafx_initialize(void *data) do_log(LOG_ERROR, "NvAFX_SetU32(Sample Rate: %f) failed, error %i", ng->sample_rate, err); - ng->use_nvafx = false; - return false; + goto failure; } // initial setting of intensity to 1.0f err = NvAFX_SetFloat( @@ -237,8 +258,7 @@ static inline bool nvafx_initialize(void *data) do_log(LOG_ERROR, "NvAFX_SetFloat(Intensity Ratio: %f) failed, error %i", 1.0f, err); - ng->use_nvafx = false; - return false; + goto failure; } err = NvAFX_SetString(ng->handle[i], NVAFX_PARAM_DENOISER_MODEL_PATH, @@ -247,16 +267,14 @@ static inline bool nvafx_initialize(void *data) do_log(LOG_ERROR, "NvAFX_SetString() failed, error %i", err); - ng->use_nvafx = false; - return false; + goto failure; } err = NvAFX_Load(ng->handle[i]); if (err != NVAFX_STATUS_SUCCESS) { do_log(LOG_ERROR, "NvAFX_Load() failed with error %i", err); - ng->use_nvafx = false; - return false; + goto failure; } } if (ng->use_nvafx) { @@ -267,14 +285,12 @@ static inline bool nvafx_initialize(void *data) do_log(LOG_ERROR, "NvAFX_GetU32() failed to get the number of channels, error %i", err); - ng->use_nvafx = false; - return false; + goto failure; } if (ng->num_channels != 1) { do_log(LOG_ERROR, "The number of channels is not 1 in the sdk any more ==> update code"); - ng->use_nvafx = false; - return false; + goto failure; } NvAFX_Status err = NvAFX_GetU32( ng->handle[0], @@ -284,21 +300,29 @@ static inline bool nvafx_initialize(void *data) do_log(LOG_ERROR, "NvAFX_GetU32() failed to get the number of samples per frame, error %i", err); - ng->use_nvafx = false; - return false; + goto failure; } if (ng->num_samples_per_frame != NVAFX_FRAME_SIZE) { do_log(LOG_ERROR, "The number of samples per frame has changed from 480 (= 10 ms) ==> update code"); - ng->use_nvafx = false; - return false; + goto failure; } } } - return true; + ng->nvafx_initialized = true; + pthread_mutex_unlock(&nvafx_initializer_mutex); + pthread_mutex_unlock(&ng->nvafx_mutex); + return NULL; + +failure: + ng->use_nvafx = false; + pthread_mutex_unlock(&nvafx_initializer_mutex); + pthread_mutex_unlock(&ng->nvafx_mutex); + return NULL; + #else UNUSED_PARAMETER(data); - return false; + return NULL; #endif } @@ -365,20 +389,24 @@ static void noise_suppress_update(void *data, obs_data_t *s) #ifdef LIBNVAFX_ENABLED ng->intensity_ratio = (float)obs_data_get_double(s, S_NVAFX_INTENSITY); - if (ng->nvafx_initialized && ng->use_nvafx) { - int err; - for (int i = 0; i < MAX_PREPROC_CHANNELS; i++) { - err = NvAFX_SetFloat( - ng->handle[i], - NVAFX_PARAM_DENOISER_INTENSITY_RATIO, - ng->intensity_ratio); - if (err != NVAFX_STATUS_SUCCESS) { - do_log(LOG_ERROR, - "NvAFX_SetFloat(Intensity Ratio: %f) failed, error %i", - ng->intensity_ratio, err); - ng->use_nvafx = false; + if (ng->use_nvafx) { + pthread_mutex_lock(&ng->nvafx_mutex); + if (ng->nvafx_initialized) { + int err; + for (int i = 0; i < MAX_PREPROC_CHANNELS; i++) { + err = NvAFX_SetFloat( + ng->handle[i], + NVAFX_PARAM_DENOISER_INTENSITY_RATIO, + ng->intensity_ratio); + if (err != NVAFX_STATUS_SUCCESS) { + do_log(LOG_ERROR, + "NvAFX_SetFloat(Intensity Ratio: %f) failed, error %i", + ng->intensity_ratio, err); + ng->use_nvafx = false; + } } } + pthread_mutex_unlock(&ng->nvafx_mutex); } #endif /* Ignore if already allocated */ @@ -387,7 +415,7 @@ static void noise_suppress_update(void *data, obs_data_t *s) return; #endif #ifdef LIBNVAFX_ENABLED - if (ng->use_nvafx && ng->nvafx_initialized) + if (ng->use_nvafx && (ng->nvafx_initialized || ng->nvafx_loading)) return; #endif #ifdef LIBRNNOISE_ENABLED @@ -425,8 +453,10 @@ static void noise_suppress_update(void *data, obs_data_t *s) } #ifdef LIBNVAFX_ENABLED - if (!ng->nvafx_initialized && ng->use_nvafx) - ng->nvafx_initialized = nvafx_initialize(data); + if (!ng->nvafx_initialized && ng->use_nvafx && !ng->nvafx_loading) { + ng->nvafx_loading = true; + pthread_create(&ng->nvafx_thread, NULL, nvafx_initialize, ng); + } #endif for (size_t i = 0; i < channels; i++) alloc_channel(ng, sample_rate, i, frames); @@ -484,6 +514,8 @@ bool load_nvafx(void) blog(LOG_INFO, "[noise suppress: Nvidia RTX denoiser enabled]"); } + pthread_mutex_init(&nvafx_initializer_mutex, PTHREAD_MUTEX_DEFAULT); + #define LOAD_SYM_FROM_LIB(sym, lib, dll) \ if (!(sym = (sym##_t)GetProcAddress(lib, #sym))) { \ DWORD err = GetLastError(); \ @@ -541,6 +573,9 @@ static void *noise_suppress_create(obs_data_t *settings, obs_source_t *filter) ng->model = buffer; ng->nvafx_enabled = true; ng->nvafx_initialized = false; + ng->nvafx_loading = false; + + pthread_mutex_init(&ng->nvafx_mutex, PTHREAD_MUTEX_DEFAULT); info("NVAFX SDK redist path was found here %s", sdk_path); }