From a6a6118c04bbb0b029cf30e70a2bb2a1d2ed5ecc Mon Sep 17 00:00:00 2001 From: jp9000 Date: Thu, 31 Oct 2013 10:28:47 -0700 Subject: [PATCH] finish up most of the source audio stuff and rename some variables/structs/enum to be a bit more consistent --- libobs/media-io/audio-io.c | 3 +- libobs/media-io/audio-io.h | 3 +- libobs/media-io/audio-resampler-ffmpeg.c | 7 +- libobs/media-io/audio-resampler.h | 11 +- libobs/media-io/video-io.c | 3 +- libobs/media-io/video-io.h | 21 ++-- libobs/obs-source.c | 129 ++++++++++++++++++----- libobs/obs-source.h | 16 ++- libobs/obs.c | 2 + libobs/obs.h | 10 +- test/win/test.cpp | 4 +- 11 files changed, 152 insertions(+), 57 deletions(-) diff --git a/libobs/media-io/audio-io.c b/libobs/media-io/audio-io.c index b881ff8bf..d17686975 100644 --- a/libobs/media-io/audio-io.c +++ b/libobs/media-io/audio-io.c @@ -91,7 +91,6 @@ static inline bool valid_audio_params(struct audio_info *info) static inline bool ao_add_to_media(audio_t audio) { struct media_output_info oi; - oi.format = audio->info.format; oi.obj = audio; oi.connect = NULL; @@ -117,7 +116,7 @@ int audio_output_open(audio_t *audio, media_t media, struct audio_info *info) pthread_mutex_init_value(&out->line_mutex); out->media = media; out->block_size = get_audio_channels(info->speakers) * - get_audio_bytes_per_channel(info->type); + get_audio_bytes_per_channel(info->format); if (pthread_mutex_init(&out->line_mutex, NULL) != 0) goto fail; diff --git a/libobs/media-io/audio-io.h b/libobs/media-io/audio-io.h index 61bf7d3eb..4ca08fd82 100644 --- a/libobs/media-io/audio-io.h +++ b/libobs/media-io/audio-io.h @@ -64,10 +64,9 @@ struct audio_data { struct audio_info { const char *name; - const char *format; uint32_t samples_per_sec; - enum audio_format type; + enum audio_format format; enum speaker_layout speakers; }; diff --git a/libobs/media-io/audio-resampler-ffmpeg.c b/libobs/media-io/audio-resampler-ffmpeg.c index 661100c43..dfd752656 100644 --- a/libobs/media-io/audio-resampler-ffmpeg.c +++ b/libobs/media-io/audio-resampler-ffmpeg.c @@ -124,7 +124,8 @@ void audio_resampler_destroy(audio_resampler_t rs) bool audio_resampler_resample(audio_resampler_t rs, void **output, uint32_t *out_frames, - void *input, uint32_t in_frames) + const void *input, uint32_t in_frames, + uint64_t *timestamp_offset) { struct SwrContext *context = rs->context; int ret; @@ -134,6 +135,8 @@ bool audio_resampler_resample(audio_resampler_t rs, (int64_t)rs->output_freq, (int64_t)rs->input_freq, AV_ROUND_UP); + *timestamp_offset = (uint64_t)swr_get_delay(context, 1000000000); + /* resize the buffer if bigger */ if (estimated > rs->output_size) { if (rs->output_buffer) @@ -146,7 +149,7 @@ bool audio_resampler_resample(audio_resampler_t rs, ret = swr_convert(context, &rs->output_buffer, rs->output_size, - input, in_frames); + (const uint8_t**)&input, in_frames); if (ret < 0) { blog(LOG_ERROR, "swr_convert failed: %d", ret); diff --git a/libobs/media-io/audio-resampler.h b/libobs/media-io/audio-resampler.h index 35f9f63b3..e897a988d 100644 --- a/libobs/media-io/audio-resampler.h +++ b/libobs/media-io/audio-resampler.h @@ -20,6 +20,10 @@ #include "../util/c99defs.h" #include "audio-io.h" +#ifdef __cplusplus +extern "C" { +#endif + struct audio_resampler; typedef struct audio_resampler *audio_resampler_t; @@ -35,4 +39,9 @@ EXPORT void audio_resampler_destroy(audio_resampler_t resampler); EXPORT bool audio_resampler_resample(audio_resampler_t resampler, void **output, uint32_t *out_frames, - void *input, uint32_t in_frames); + const void *input, uint32_t in_frames, + uint64_t *timestamp_offset); + +#ifdef __cplusplus +} +#endif diff --git a/libobs/media-io/video-io.c b/libobs/media-io/video-io.c index 9e03f5668..2923832ad 100644 --- a/libobs/media-io/video-io.c +++ b/libobs/media-io/video-io.c @@ -80,13 +80,12 @@ static void *video_thread(void *param) static inline bool valid_video_params(struct video_info *info) { return info->height != 0 && info->width != 0 && info->fps_den != 0 && - info->fps_num != 0 && info->format != NULL; + info->fps_num != 0; } static inline bool vo_add_to_media(video_t video) { struct media_output_info oi; - oi.format = video->info.format; oi.obj = video; oi.connect = NULL; diff --git a/libobs/media-io/video-io.h b/libobs/media-io/video-io.h index fdb2d0741..f24e3553f 100644 --- a/libobs/media-io/video-io.h +++ b/libobs/media-io/video-io.h @@ -32,7 +32,7 @@ extern "C" { struct video_output; typedef struct video_output *video_t; -enum video_type { +enum video_format { VIDEO_FORMAT_UNKNOWN, /* planar 420 format */ @@ -53,20 +53,19 @@ enum video_type { }; struct video_frame { - const void *data; - uint32_t row_size; /* for RGB/BGR formats and UYVX */ - uint64_t timestamp; + const void *data; + uint32_t row_size; /* for RGB/BGR formats and UYVX */ + uint64_t timestamp; }; struct video_info { - const char *name; - const char *format; + const char *name; - enum video_type type; - uint32_t fps_num; /* numerator */ - uint32_t fps_den; /* denominator */ - uint32_t width; - uint32_t height; + enum video_format type; + uint32_t fps_num; /* numerator */ + uint32_t fps_den; /* denominator */ + uint32_t width; + uint32_t height; }; #define VIDEO_OUTPUT_SUCCESS 0 diff --git a/libobs/obs-source.c b/libobs/obs-source.c index eacc78049..a82523be0 100644 --- a/libobs/obs-source.c +++ b/libobs/obs-source.c @@ -171,8 +171,8 @@ void obs_source_destroy(obs_source_t source) pthread_mutex_unlock(&obs->source_list_mutex); } - for (i = 0; i < source->audio_buffer.num; i++) - audiobuf_free(source->audio_buffer.array+i); + for (i = 0; i < source->audio_wait_buffer.num; i++) + audiobuf_free(source->audio_wait_buffer.array+i); for (i = 0; i < source->video_frames.num; i++) source_frame_destroy(source->video_frames.array[i]); @@ -183,10 +183,12 @@ void obs_source_destroy(obs_source_t source) if (source->data) source->callbacks.destroy(source->data); + bfree(source->audio_data.data); audio_line_destroy(source->audio_line); + audio_resampler_destroy(source->resampler); da_free(source->video_frames); - da_free(source->audio_buffer); + da_free(source->audio_wait_buffer); da_free(source->filters); pthread_mutex_destroy(&source->filter_mutex); pthread_mutex_destroy(&source->audio_mutex); @@ -259,15 +261,15 @@ static void source_output_audio_line(obs_source_t source, audio_line_output(source->audio_line, &in); } -static void obs_source_flush_audio_buffer(obs_source_t source) +static void obs_source_flush_audio_wait_buffer(obs_source_t source) { size_t i; pthread_mutex_lock(&source->audio_mutex); source->timing_set = true; - for (i = 0; i < source->audio_buffer.num; i++) { - struct audiobuf *buf = source->audio_buffer.array+i; + for (i = 0; i < source->audio_wait_buffer.num; i++) { + struct audiobuf *buf = source->audio_wait_buffer.array+i; struct audio_data data; data.data = buf->data; @@ -277,7 +279,7 @@ static void obs_source_flush_audio_buffer(obs_source_t source) audiobuf_free(buf); } - da_free(source->audio_buffer); + da_free(source->audio_wait_buffer); pthread_mutex_unlock(&source->audio_mutex); } @@ -306,9 +308,9 @@ enum convert_type { CONVERT_422_Y, }; -static inline enum convert_type get_convert_type(enum video_type type) +static inline enum convert_type get_convert_type(enum video_format format) { - switch (type) { + switch (format) { case VIDEO_FORMAT_I420: return CONVERT_420; case VIDEO_FORMAT_NV12: @@ -332,9 +334,9 @@ static inline enum convert_type get_convert_type(enum video_type type) return CONVERT_NONE; } -static inline bool is_yuv(enum video_type type) +static inline bool is_yuv(enum video_format format) { - switch (type) { + switch (format) { case VIDEO_FORMAT_I420: case VIDEO_FORMAT_NV12: case VIDEO_FORMAT_YVYU: @@ -357,7 +359,7 @@ static bool upload_frame(texture_t tex, const struct source_frame *frame) { void *ptr; uint32_t row_bytes; - enum convert_type type = get_convert_type(frame->type); + enum convert_type type = get_convert_type(frame->format); if (type == CONVERT_NONE) { texture_setimage(tex, frame->data, frame->row_bytes, false); @@ -390,7 +392,7 @@ static bool upload_frame(texture_t tex, const struct source_frame *frame) static void obs_source_draw_texture(texture_t tex, struct source_frame *frame) { effect_t effect = obs->default_effect; - bool yuv = is_yuv(frame->type); + bool yuv = is_yuv(frame->format); const char *type = yuv ? "DrawYUVToRGB" : "DrawRGB"; technique_t tech; eparam_t param; @@ -424,8 +426,8 @@ static void obs_source_render_async_video(obs_source_t source) return; source->timing_adjust = frame->timestamp - os_gettime_ns(); - if (!source->timing_set && source->audio_buffer.num) - obs_source_flush_audio_buffer(source); + if (!source->timing_set && source->audio_wait_buffer.num) + obs_source_flush_audio_wait_buffer(source); if (set_texture_size(source, frame)) obs_source_draw_texture(source->output_texture, frame); @@ -631,8 +633,8 @@ void obs_source_output_video(obs_source_t source, } } -static inline const struct audio_data *filter_async_audio(obs_source_t source, - const struct audio_data *in) +static inline struct filtered_audio *filter_async_audio(obs_source_t source, + struct filtered_audio *in) { size_t i; for (i = source->filters.num; i > 0; i--) { @@ -647,42 +649,113 @@ static inline const struct audio_data *filter_async_audio(obs_source_t source, return in; } -static struct audio_data *process_audio(obs_source_t source, +static inline void reset_resampler(obs_source_t source, const struct source_audio *audio) { - /* TODO: upmix/downmix/resample */ - return NULL; + const struct audio_info *obs_info = audio_output_getinfo(obs->audio); + struct resample_info output_info; + + output_info.format = obs_info->format; + output_info.samples_per_sec = obs_info->samples_per_sec; + output_info.speakers = obs_info->speakers; + + source->sample_info.format = audio->format; + source->sample_info.samples_per_sec = audio->samples_per_sec; + source->sample_info.speakers = audio->speakers; + + if (source->sample_info.samples_per_sec == obs_info->samples_per_sec && + source->sample_info.format == obs_info->format && + source->sample_info.speakers == obs_info->speakers) { + source->audio_failed = false; + return; + } + + audio_resampler_destroy(source->resampler); + source->resampler = audio_resampler_create(&output_info, + &source->sample_info); + + source->audio_failed = source->resampler == NULL; + if (source->resampler == NULL) + blog(LOG_ERROR, "creation of resampler failed"); +} + +static inline void copy_audio_data(obs_source_t source, + const void *data, uint32_t frames, uint64_t timestamp) +{ + size_t blocksize = audio_output_blocksize(obs->audio); + size_t size = (size_t)frames * blocksize; + + /* ensure audio storage capacity */ + if (source->audio_storage_size < size) { + bfree(source->audio_data.data); + source->audio_data.data = bmalloc(size); + source->audio_storage_size = size; + } + + source->audio_data.frames = frames; + source->audio_data.timestamp = timestamp; + memcpy(source->audio_data.data, data, size); +} + +/* resamples/remixes new audio to the designated main audio output format */ +static void process_audio(obs_source_t source, const struct source_audio *audio) +{ + if (source->sample_info.samples_per_sec != audio->samples_per_sec || + source->sample_info.format != audio->format || + source->sample_info.speakers != audio->speakers) + reset_resampler(source, audio); + + if (source->audio_failed) + return; + + if (source->resampler) { + void *output; + uint32_t frames; + uint64_t offset; + + audio_resampler_resample(source->resampler, &output, &frames, + audio->data, audio->frames, &offset); + + copy_audio_data(source, output, frames, + audio->timestamp - offset); + } else { + copy_audio_data(source, audio->data, audio->frames, + audio->timestamp); + } } void obs_source_output_audio(obs_source_t source, const struct source_audio *audio) { uint32_t flags = obs_source_get_output_flags(source); - struct audio_data *data; - const struct audio_data *output; + size_t blocksize = audio_output_blocksize(obs->audio); + struct filtered_audio *output; - data = process_audio(source, audio); + process_audio(source, audio); pthread_mutex_lock(&source->filter_mutex); - output = filter_async_audio(source, data); + output = filter_async_audio(source, &source->audio_data); if (output) { pthread_mutex_lock(&source->audio_mutex); if (!source->timing_set && flags & SOURCE_ASYNC_VIDEO) { struct audiobuf newbuf; - size_t audio_size = audio_output_blocksize(obs->audio) * - output->frames; + size_t audio_size = blocksize * output->frames; newbuf.data = bmalloc(audio_size); newbuf.frames = output->frames; newbuf.timestamp = output->timestamp; memcpy(newbuf.data, output->data, audio_size); - da_push_back(source->audio_buffer, &newbuf); + da_push_back(source->audio_wait_buffer, &newbuf); } else { - source_output_audio_line(source, output); + struct audio_data data; + data.data = output->data; + data.frames = output->frames; + data.timestamp = output->timestamp; + source_output_audio_line(source, &data); } pthread_mutex_unlock(&source->audio_mutex); diff --git a/libobs/obs-source.h b/libobs/obs-source.h index b68e9d350..6ebf9665c 100644 --- a/libobs/obs-source.h +++ b/libobs/obs-source.h @@ -22,6 +22,7 @@ #include "util/dstr.h" #include "util/threading.h" #include "media-io/media-io.h" +#include "media-io/audio-resampler.h" /* * =========================================== @@ -144,8 +145,8 @@ * returns: New video frame data (or NULL if pending) * * --------------------------------------------------------- - * const struct audio_data *[name]_filter_audio(void *data, - * const struct audio_data *audio); + * struct filter_audio [name]_filter_audio(void *data, + * struct filter_audio *audio); * Filters video data. Used with async video data. * * audio: Audio data. @@ -187,8 +188,8 @@ struct source_info { struct source_frame *(*filter_video)(void *data, const struct source_frame *frame); - const struct audio_data *(*filter_audio)(void *data, - const struct audio_data *audio); + struct filtered_audio *(*filter_audio)(void *data, + struct filtered_audio *audio); }; struct audiobuf { @@ -215,11 +216,16 @@ struct obs_source { uint64_t last_sys_timestamp; texture_t output_texture; + bool audio_failed; + struct resample_info sample_info; + audio_resampler_t resampler; audio_line_t audio_line; - DARRAY(struct audiobuf) audio_buffer; + DARRAY(struct audiobuf) audio_wait_buffer; DARRAY(struct source_frame*) video_frames; pthread_mutex_t audio_mutex; pthread_mutex_t video_mutex; + struct filtered_audio audio_data; + size_t audio_storage_size; /* filters */ struct obs_source *filter_parent; diff --git a/libobs/obs.c b/libobs/obs.c index d5f4de821..b097a0889 100644 --- a/libobs/obs.c +++ b/libobs/obs.c @@ -118,6 +118,8 @@ static inline void obs_free_graphics(void) for (i = 0; i < NUM_TEXTURES; i++) stagesurface_destroy(obs->copy_surfaces[i]); + effect_destroy(obs->default_effect); + gs_leavecontext(); gs_destroy(obs->graphics); diff --git a/libobs/obs.h b/libobs/obs.h index 1587404b0..1931d20f2 100644 --- a/libobs/obs.h +++ b/libobs/obs.h @@ -48,13 +48,19 @@ enum order_movement { ORDER_MOVE_BOTTOM }; +struct filtered_audio { + void *data; + uint32_t frames; + uint64_t timestamp; +}; + struct source_audio { const void *data; uint32_t frames; /* audio will be automatically resampled/upmixed/downmixed */ enum speaker_layout speakers; - enum audio_format type; + enum audio_format format; uint32_t samples_per_sec; /* can be 0 if 'immediate' */ @@ -68,7 +74,7 @@ struct source_frame { uint32_t row_bytes; uint64_t timestamp; - enum video_type type; + enum video_format format; float yuv_matrix[16]; bool flip; }; diff --git a/test/win/test.cpp b/test/win/test.cpp index 600715760..f73d94fcf 100644 --- a/test/win/test.cpp +++ b/test/win/test.cpp @@ -3,6 +3,7 @@ #include #include "util/base.h" +#include "media-io/audio-resampler.h" #include "obs.h" #include @@ -69,7 +70,6 @@ static void CreateOBS(HWND hwnd) struct video_info vi; memset(&vi, 0, sizeof(struct video_info)); - vi.format = "RGBA"; vi.fps_num = 30000; vi.fps_den = 1001; vi.width = rc.right; @@ -111,7 +111,6 @@ static void AddTestItems(obs_scene_t scene, obs_source_t source) static HWND CreateTestWindow(HINSTANCE instance) { WNDCLASS wc; - base_set_log_handler(do_log); memset(&wc, 0, sizeof(wc)); wc.lpszClassName = L"bla"; @@ -134,6 +133,7 @@ int WINAPI WinMain(HINSTANCE instance, HINSTANCE prevInstance, LPSTR cmdLine, int numCmd) { HWND hwnd = NULL; + base_set_log_handler(do_log); try { hwnd = CreateTestWindow(instance);