From 63a131ce22d14f71949d3181e1266fe711b233dd Mon Sep 17 00:00:00 2001 From: PatTheMav Date: Thu, 30 Nov 2023 15:40:12 +0100 Subject: [PATCH] mac-videotoolbox: Fix handling of unsuccessful encoder creation When an encoder was not created in create_encoder, the appropriate OSStatus value is returned but the calling code expects a boolean return value. The negative OSStatus code sent on error is thus interpreted as a truthy value and the error is not detected. Changing the call signature to correctly return an OSStatus (and change the caller to detect error situations) fixes this. --- plugins/mac-videotoolbox/encoder.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/mac-videotoolbox/encoder.c b/plugins/mac-videotoolbox/encoder.c index b42e1f3c5..9603634a0 100644 --- a/plugins/mac-videotoolbox/encoder.c +++ b/plugins/mac-videotoolbox/encoder.c @@ -537,7 +537,7 @@ static inline CFDictionaryRef create_pixbuf_spec(struct vt_encoder *enc) return pixbuf_spec; } -static bool create_encoder(struct vt_encoder *enc) +static OSStatus create_encoder(struct vt_encoder *enc) { OSStatus code; @@ -678,7 +678,7 @@ static bool create_encoder(struct vt_encoder *enc) enc->session = s; - return true; + return noErr; } static void vt_destroy(void *data) @@ -877,8 +877,10 @@ static void *vt_create(obs_data_t *settings, obs_encoder_t *encoder) goto fail; } - if (!create_encoder(enc)) + code = create_encoder(enc); + if (code != noErr) { goto fail; + } dump_encoder_info(enc);