From 6b8e84844a1a6d0b223b857725057be88f142cec Mon Sep 17 00:00:00 2001 From: jp9000 Date: Thu, 9 Jan 2014 19:08:20 -0700 Subject: [PATCH] Add preliminary audio processing - Mixing still isn't implemented, but the audio system should be able to start up, and mix at least once audio line for the time being. Will have to write some test audio sources to verify things are working properly, and build the rest of the output functionality. --- libobs/media-io/audio-io.c | 162 ++++++++++++++++++++++++++++++++----- libobs/media-io/audio-io.h | 4 +- libobs/obs-source.c | 5 +- libobs/obs.c | 7 +- libobs/util/circlebuf.h | 15 ++-- obs/window-basic-main.cpp | 18 ++++- obs/window-basic-main.hpp | 1 + 7 files changed, 174 insertions(+), 38 deletions(-) diff --git a/libobs/media-io/audio-io.c b/libobs/media-io/audio-io.c index 13aef2aaf..7d4677ed1 100644 --- a/libobs/media-io/audio-io.c +++ b/libobs/media-io/audio-io.c @@ -25,8 +25,11 @@ /* TODO: Incomplete */ struct audio_line { + char *name; + struct audio_output *audio; struct circlebuf buffer; + pthread_mutex_t mutex; DARRAY(uint8_t) volume_buffer; uint64_t base_timestamp; uint64_t last_timestamp; @@ -34,12 +37,17 @@ struct audio_line { /* states whether this line is still being used. if not, then when the * buffer is depleted, it's destroyed */ bool alive; + + struct audio_line **prev_next; + struct audio_line *next; }; static inline void audio_line_destroy_data(struct audio_line *line) { circlebuf_free(&line->buffer); da_free(line->volume_buffer); + pthread_mutex_destroy(&line->mutex); + bfree(line->name); bfree(line); } @@ -55,29 +63,113 @@ struct audio_output { DARRAY(uint8_t) pending_bytes; + DARRAY(uint8_t) mix_buffer; + bool initialized; pthread_mutex_t line_mutex; - DARRAY(struct audio_line*) lines; + struct audio_line *first_line; }; static inline void audio_output_removeline(struct audio_output *audio, struct audio_line *line) { pthread_mutex_lock(&audio->line_mutex); - da_erase_item(audio->lines, &line); + *line->prev_next = line->next; pthread_mutex_unlock(&audio->line_mutex); + audio_line_destroy_data(line); } +static inline size_t convert_to_sample_offset(audio_t audio, uint64_t offset) +{ + double audio_offset_d = (double)offset; + audio_offset_d /= 1000000000.0; + audio_offset_d *= (double)audio->info.samples_per_sec; + + return (size_t)audio_offset_d * audio->block_size; +} + /* ------------------------------------------------------------------------- */ +static inline void clear_excess_audio_data(struct audio_line *line, + uint64_t size) +{ + if (size > line->buffer.size) + size = line->buffer.size; + + blog(LOG_WARNING, "Excess audio data for audio line '%s', somehow " + "audio data went back in time by %llu bytes", + line->name, size); + + circlebuf_pop_front(&line->buffer, NULL, (size_t)size); +} + +static inline uint64_t min_uint64(uint64_t a, uint64_t b) +{ + return a < b ? a : b; +} + +static inline void mix_audio_line(struct audio_output *audio, + struct audio_line *line, size_t size) +{ + /* TODO: this just overwrites, handle actual mixing */ + circlebuf_pop_front(&line->buffer, audio->mix_buffer.array, size); +} + +static void mix_audio_lines(struct audio_output *audio, uint64_t audio_time, + uint64_t prev_time) +{ + struct audio_line *line = audio->first_line; + uint64_t time_offset = audio_time - prev_time; + uint64_t byte_offset = convert_to_sample_offset(audio, time_offset); + + da_resize(audio->mix_buffer, byte_offset); + memset(audio->mix_buffer.array, 0, byte_offset); + + while (line) { + struct audio_line *next = line->next; + + if (line->base_timestamp < prev_time) { + clear_excess_audio_data(line, + line->base_timestamp - prev_time); + line->base_timestamp = prev_time; + } + + size_t pop_size = min_uint64(byte_offset, line->buffer.size); + if (pop_size) + mix_audio_line(audio, line, pop_size); + else if (!line->alive) + audio_output_removeline(audio, line); + + line = next; + } + + /* TODO - not good enough */ + /*if (audio->output) + media_output_data(audio->output, audio->mix_buffer.array);*/ +} + +/* sample audio 40 times a second */ +#define AUDIO_WAIT_TIME (1000/40) + static void *audio_thread(void *param) { struct audio_output *audio = param; + uint64_t buffer_time = audio->info.buffer_ms * 1000000; + uint64_t prev_time = os_gettime_ns() - buffer_time; + uint64_t audio_time; while (event_try(&audio->stop_event) == EAGAIN) { - /* TODO */ + os_sleep_ms(AUDIO_WAIT_TIME); + + pthread_mutex_lock(&audio->line_mutex); + + audio_time = os_gettime_ns() - buffer_time; + mix_audio_lines(audio, audio_time, prev_time); + prev_time = audio_time; + + pthread_mutex_unlock(&audio->line_mutex); } return NULL; @@ -91,7 +183,7 @@ static inline bool valid_audio_params(struct audio_info *info) info->speakers > 0; } -static inline bool ao_add_to_media(audio_t audio) +static bool ao_add_to_media(audio_t audio) { struct media_output_info oi; oi.obj = audio; @@ -109,6 +201,7 @@ static inline bool ao_add_to_media(audio_t audio) int audio_output_open(audio_t *audio, media_t media, struct audio_info *info) { struct audio_output *out; + pthread_mutexattr_t attr; if (!valid_audio_params(info)) return AUDIO_OUTPUT_INVALIDPARAM; @@ -123,7 +216,11 @@ int audio_output_open(audio_t *audio, media_t media, struct audio_info *info) out->block_size = out->channels * get_audio_bytes_per_channel(info->format); - if (pthread_mutex_init(&out->line_mutex, NULL) != 0) + if (pthread_mutexattr_init(&attr) != 0) + goto fail; + if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) != 0) + goto fail; + if (pthread_mutex_init(&out->line_mutex, &attr) != 0) goto fail; if (event_init(&out->stop_event, true) != 0) goto fail; @@ -141,15 +238,32 @@ fail: return AUDIO_OUTPUT_FAIL; } -audio_line_t audio_output_createline(audio_t audio) +audio_line_t audio_output_createline(audio_t audio, const char *name) { struct audio_line *line = bmalloc(sizeof(struct audio_line)); memset(line, 0, sizeof(struct audio_line)); line->alive = true; + if (pthread_mutex_init(&line->mutex, NULL) != 0) { + blog(LOG_ERROR, "audio_output_createline: Failed to create " + "mutex"); + bfree(line); + return NULL; + } + pthread_mutex_lock(&audio->line_mutex); - da_push_back(audio->lines, &line); + + if (audio->first_line) { + audio->first_line->prev_next = &line->next; + line->next = audio->first_line; + } + + line->prev_next = &audio->first_line; + audio->first_line = line; + pthread_mutex_unlock(&audio->line_mutex); + + line->name = bstrdup(name ? name : "(unnamed audio line)"); return line; } @@ -161,7 +275,7 @@ const struct audio_info *audio_output_getinfo(audio_t audio) void audio_output_close(audio_t audio) { void *thread_ret; - size_t i; + struct audio_line *line; if (!audio) return; @@ -171,11 +285,17 @@ void audio_output_close(audio_t audio) pthread_join(audio->thread, &thread_ret); } - for (i = 0; i < audio->lines.num; i++) - audio_line_destroy_data(audio->lines.array[i]); + line = audio->first_line; + while (line) { + struct audio_line *next = line->next; + audio_line_destroy_data(line); + line = next; + } - da_free(audio->lines); + da_free(audio->mix_buffer); + da_free(audio->pending_bytes); media_remove_output(audio->media, audio->output); + media_output_destroy(audio->output); event_destroy(&audio->stop_event); pthread_mutex_destroy(&audio->line_mutex); bfree(audio); @@ -196,12 +316,6 @@ size_t audio_output_blocksize(audio_t audio) return audio->block_size; } -static inline uint64_t convert_to_sample_offset(audio_t audio, uint64_t offset) -{ - return (uint64_t)((double)offset * - (1000000000.0 / (double)audio->info.samples_per_sec)); -} - static inline void mul_vol_u8bit(struct audio_line *line, float volume, size_t total_num) { @@ -307,15 +421,21 @@ static void audio_line_place_data(struct audio_line *line, void audio_line_output(audio_line_t line, const struct audio_data *data) { + /* TODO: prevent insertation of data too far away from expected + * audio timing */ + + pthread_mutex_lock(&line->mutex); + if (!line->buffer.size) { line->base_timestamp = data->timestamp; audio_line_place_data(line, data, 0); } else { - uint64_t position = data->timestamp - line->base_timestamp; - position = convert_to_sample_offset(line->audio, position); - position *= line->audio->block_size; + uint64_t time_offset = data->timestamp - line->base_timestamp; + size_t pos = convert_to_sample_offset(line->audio, time_offset); - audio_line_place_data(line, data, (size_t)position); + audio_line_place_data(line, data, pos); } + + pthread_mutex_unlock(&line->mutex); } diff --git a/libobs/media-io/audio-io.h b/libobs/media-io/audio-io.h index 81553f5ce..e743fcda0 100644 --- a/libobs/media-io/audio-io.h +++ b/libobs/media-io/audio-io.h @@ -69,6 +69,8 @@ struct audio_info { uint32_t samples_per_sec; enum audio_format format; enum speaker_layout speakers; + + uint64_t buffer_ms; }; static inline uint32_t get_audio_channels(enum speaker_layout speakers) @@ -117,7 +119,7 @@ static inline size_t get_audio_size(enum audio_format type, EXPORT int audio_output_open(audio_t *audio, media_t media, struct audio_info *info); -EXPORT audio_line_t audio_output_createline(audio_t audio); +EXPORT audio_line_t audio_output_createline(audio_t audio, const char *name); EXPORT size_t audio_output_blocksize(audio_t audio); EXPORT const struct audio_info *audio_output_getinfo(audio_t audio); EXPORT void audio_output_close(audio_t audio); diff --git a/libobs/obs-source.c b/libobs/obs-source.c index 64a22fbfe..baf6bd9f6 100644 --- a/libobs/obs-source.c +++ b/libobs/obs-source.c @@ -143,10 +143,11 @@ bool obs_source_init(struct obs_source *source, const char *settings, return false; if (flags & SOURCE_AUDIO) { - source->audio_line = audio_output_createline(obs->audio.audio); + source->audio_line = audio_output_createline(obs->audio.audio, + source->name); if (!source->audio_line) { blog(LOG_ERROR, "Failed to create audio line for " - "source"); + "source '%s'", source->name); return false; } } diff --git a/libobs/obs.c b/libobs/obs.c index 01f141425..27d82cf2b 100644 --- a/libobs/obs.c +++ b/libobs/obs.c @@ -355,14 +355,11 @@ bool obs_reset_video(struct obs_video_info *ovi) bool obs_reset_audio(struct audio_info *ai) { - /*obs_free_audio(); + obs_free_audio(); if(!ai) return true; - return obs_init_audio(ai);*/ - - /* TODO */ - return true; + return obs_init_audio(ai); } bool obs_get_video_info(struct obs_video_info *ovi) diff --git a/libobs/util/circlebuf.h b/libobs/util/circlebuf.h index 9847fc578..deda4cfd9 100644 --- a/libobs/util/circlebuf.h +++ b/libobs/util/circlebuf.h @@ -176,12 +176,15 @@ static inline void circlebuf_pop_front(struct circlebuf *cb, void *data, start_size = cb->capacity - cb->start_pos; - if (start_size < size) { - memcpy(data, (uint8_t*)cb->data + cb->start_pos, start_size); - memcpy((uint8_t*)data + start_size, cb->data, - size - start_size); - } else { - memcpy(data, (uint8_t*)cb->data + cb->start_pos, size); + if (data) { + if (start_size < size) { + memcpy(data, (uint8_t*)cb->data + cb->start_pos, + start_size); + memcpy((uint8_t*)data + start_size, cb->data, + size - start_size); + } else { + memcpy(data, (uint8_t*)cb->data + cb->start_pos, size); + } } cb->size -= size; diff --git a/obs/window-basic-main.cpp b/obs/window-basic-main.cpp index f117d1d4a..8a47d4823 100644 --- a/obs/window-basic-main.cpp +++ b/obs/window-basic-main.cpp @@ -199,6 +199,8 @@ bool OBSBasic::Init() return false; if (!InitGraphics()) return false; + if (!InitAudio()) + return false; signal_handler_connect(obs_signalhandler(), "source-add", OBSBasic::SourceAdded, this); @@ -243,10 +245,20 @@ bool OBSBasic::InitGraphics() ovi.window_width = size.x; ovi.window_height = size.y; - if (!obs_reset_video(&ovi)) - return false; + return obs_reset_video(&ovi); +} - return true; +bool OBSBasic::InitAudio() +{ + /* TODO: load audio settings from config */ + struct audio_info ai; + ai.name = "test"; + ai.samples_per_sec = 44100; + ai.format = AUDIO_FORMAT_16BIT; + ai.speakers = SPEAKERS_STEREO; + ai.buffer_ms = 700; + + return obs_reset_audio(&ai); } void OBSBasic::OnClose(wxCloseEvent &event) diff --git a/obs/window-basic-main.hpp b/obs/window-basic-main.hpp index 3a2f36165..8afb07db4 100644 --- a/obs/window-basic-main.hpp +++ b/obs/window-basic-main.hpp @@ -47,6 +47,7 @@ class OBSBasic : public OBSBasicBase { void AddSourcePopupMenu(); bool InitGraphics(); + bool InitAudio(); void NewProject(); void SaveProject();