From 189fc7ab6aa6280c0faee6559833a71357e92d61 Mon Sep 17 00:00:00 2001 From: jp9000 Date: Sun, 16 Aug 2020 02:20:18 -0700 Subject: [PATCH] obs-outputs: Add support for metadata-based multitrack --- plugins/obs-outputs/flv-mux.c | 251 ++++++++++++++++++++++++++++ plugins/obs-outputs/flv-mux.h | 6 + plugins/obs-outputs/rtmp-stream.c | 60 +++++-- plugins/obs-outputs/rtmp-stream.h | 2 + plugins/rtmp-services/rtmp-common.c | 13 ++ 5 files changed, 321 insertions(+), 11 deletions(-) diff --git a/plugins/obs-outputs/flv-mux.c b/plugins/obs-outputs/flv-mux.c index 40659960a..01e6caba6 100644 --- a/plugins/obs-outputs/flv-mux.c +++ b/plugins/obs-outputs/flv-mux.c @@ -268,3 +268,254 @@ void flv_packet_mux(struct encoder_packet *packet, int32_t dts_offset, *output = data.bytes.array; *size = data.bytes.num; } + +/* ------------------------------------------------------------------------- */ +/* stuff for additional media streams */ + +#define s_amf_conststring(s, str) \ + do { \ + const size_t len = sizeof(str) - 1; \ + s_wb16(s, (uint16_t)len); \ + serialize(s, str, len); \ + } while (false) + +#define s_amf_double(s, d) \ + do { \ + double d_val = d; \ + uint64_t u_val = *(uint64_t *)&d_val; \ + s_wb64(s, u_val); \ + } while (false) + +static void flv_build_additional_meta_data(uint8_t **data, size_t *size) +{ + struct array_output_data out; + struct serializer s; + + array_output_serializer_init(&s, &out); + + s_w8(&s, AMF_STRING); + s_amf_conststring(&s, "@setDataFrame"); + + s_w8(&s, AMF_STRING); + s_amf_conststring(&s, "onExpectAdditionalMedia"); + + s_w8(&s, AMF_OBJECT); + { + s_amf_conststring(&s, "processingIntents"); + + s_w8(&s, AMF_STRICT_ARRAY); + s_wb32(&s, 1); + { + s_w8(&s, AMF_STRING); + s_amf_conststring(&s, "ArchiveProgramNarrationAudio"); + } + + /* ---- */ + + s_amf_conststring(&s, "additionalMedia"); + + s_w8(&s, AMF_OBJECT); + { + s_amf_conststring(&s, "stream0"); + + s_w8(&s, AMF_OBJECT); + { + s_amf_conststring(&s, "type"); + + s_w8(&s, AMF_NUMBER); + s_amf_double(&s, RTMP_PACKET_TYPE_AUDIO); + + /* ---- */ + + s_amf_conststring(&s, "mediaLabels"); + + s_w8(&s, AMF_OBJECT); + { + s_amf_conststring(&s, "contentType"); + + s_w8(&s, AMF_STRING); + s_amf_conststring(&s, "PNAR"); + } + s_wb24(&s, AMF_OBJECT_END); + } + s_wb24(&s, AMF_OBJECT_END); + } + s_wb24(&s, AMF_OBJECT_END); + + /* ---- */ + + s_amf_conststring(&s, "defaultMedia"); + + s_w8(&s, AMF_OBJECT); + { + s_amf_conststring(&s, "audio"); + + s_w8(&s, AMF_OBJECT); + { + s_amf_conststring(&s, "mediaLabels"); + + s_w8(&s, AMF_OBJECT); + { + s_amf_conststring(&s, "contentType"); + + s_w8(&s, AMF_STRING); + s_amf_conststring(&s, "PRM"); + } + s_wb24(&s, AMF_OBJECT_END); + } + s_wb24(&s, AMF_OBJECT_END); + } + s_wb24(&s, AMF_OBJECT_END); + } + s_wb24(&s, AMF_OBJECT_END); + + *data = out.bytes.array; + *size = out.bytes.num; +} + +bool flv_additional_meta_data(obs_output_t *context, uint8_t **data, + size_t *size) +{ + obs_encoder_t *aencoder = obs_output_get_audio_encoder(context, 1); + struct array_output_data out; + struct serializer s; + uint8_t *meta_data = NULL; + size_t meta_data_size; + + if (!aencoder) + return false; + + flv_build_additional_meta_data(&meta_data, &meta_data_size); + + array_output_serializer_init(&s, &out); + + s_w8(&s, RTMP_PACKET_TYPE_INFO); //18 + + s_wb24(&s, (uint32_t)meta_data_size); + s_wb32(&s, 0); + s_wb24(&s, 0); + + s_write(&s, meta_data, meta_data_size); + bfree(meta_data); + + s_wb32(&s, (uint32_t)serializer_get_pos(&s) - 1); + + *data = out.bytes.array; + *size = out.bytes.num; + return false; +} + +static inline void s_u29(struct serializer *s, uint32_t val) +{ + if (val <= 0x7F) { + s_w8(s, val); + } else if (val <= 0x3FFF) { + s_w8(s, 0x80 | (val >> 7)); + s_w8(s, val & 0x7F); + } else if (val <= 0x1FFFFF) { + s_w8(s, 0x80 | (val >> 14)); + s_w8(s, 0x80 | ((val >> 7) & 0x7F)); + s_w8(s, val & 0x7F); + } else { + s_w8(s, 0x80 | (val >> 22)); + s_w8(s, 0x80 | ((val >> 15) & 0x7F)); + s_w8(s, 0x80 | ((val >> 8) & 0x7F)); + s_w8(s, val & 0xFF); + } +} + +static inline void s_u29b_value(struct serializer *s, uint32_t val) +{ + s_u29(s, 1 | ((val & 0xFFFFFFF) << 1)); +} + +static void flv_build_additional_audio(uint8_t **data, size_t *size, + struct encoder_packet *packet, + bool is_header, size_t index) +{ + struct array_output_data out; + struct serializer s; + + array_output_serializer_init(&s, &out); + + s_w8(&s, AMF_STRING); + s_amf_conststring(&s, "additionalMedia"); + + s_w8(&s, AMF_OBJECT); + { + s_amf_conststring(&s, "id"); + + s_w8(&s, AMF_STRING); + s_amf_conststring(&s, "stream0"); + + /* ----- */ + + s_amf_conststring(&s, "media"); + + s_w8(&s, AMF_AVMPLUS); + s_w8(&s, AMF3_BYTE_ARRAY); + s_u29b_value(&s, (uint32_t)packet->size + 2); + s_w8(&s, 0xaf); + s_w8(&s, is_header ? 0 : 1); + s_write(&s, packet->data, packet->size); + } + s_wb24(&s, AMF_OBJECT_END); + + *data = out.bytes.array; + *size = out.bytes.num; +} + +static void flv_additional_audio(struct serializer *s, int32_t dts_offset, + struct encoder_packet *packet, bool is_header, + size_t index) +{ + int32_t time_ms = get_ms_time(packet, packet->dts) - dts_offset; + uint8_t *data; + size_t size; + + if (!packet->data || !packet->size) + return; + + flv_build_additional_audio(&data, &size, packet, is_header, index); + + s_w8(s, RTMP_PACKET_TYPE_INFO); //18 + +#ifdef DEBUG_TIMESTAMPS + blog(LOG_DEBUG, "Audio2: %lu", time_ms); + + if (last_time > time_ms) + blog(LOG_DEBUG, "Non-monotonic"); + + last_time = time_ms; +#endif + + s_wb24(s, (uint32_t)size); + s_wb24(s, time_ms); + s_w8(s, (time_ms >> 24) & 0x7F); + s_wb24(s, 0); + + serialize(s, data, size); + bfree(data); + + s_wb32(s, (uint32_t)serializer_get_pos(s) - 1); +} + +void flv_additional_packet_mux(struct encoder_packet *packet, + int32_t dts_offset, uint8_t **data, size_t *size, + bool is_header, size_t index) +{ + struct array_output_data out; + struct serializer s; + + array_output_serializer_init(&s, &out); + + if (packet->type == OBS_ENCODER_VIDEO) { + //currently unsupported + bcrash("who said you could output an additional video packet?"); + } else { + flv_additional_audio(&s, dts_offset, packet, is_header, index); + } + + *data = out.bytes.array; + *size = out.bytes.num; +} diff --git a/plugins/obs-outputs/flv-mux.h b/plugins/obs-outputs/flv-mux.h index 9355491ce..3b76b8aba 100644 --- a/plugins/obs-outputs/flv-mux.h +++ b/plugins/obs-outputs/flv-mux.h @@ -30,5 +30,11 @@ extern void write_file_info(FILE *file, int64_t duration_ms, int64_t size); extern bool flv_meta_data(obs_output_t *context, uint8_t **output, size_t *size, bool write_header, size_t audio_idx); +extern bool flv_additional_meta_data(obs_output_t *context, uint8_t **output, + size_t *size); extern void flv_packet_mux(struct encoder_packet *packet, int32_t dts_offset, uint8_t **output, size_t *size, bool is_header); +extern void flv_additional_packet_mux(struct encoder_packet *packet, + int32_t dts_offset, uint8_t **output, + size_t *size, bool is_header, + size_t index); diff --git a/plugins/obs-outputs/rtmp-stream.c b/plugins/obs-outputs/rtmp-stream.c index e09ed2468..ed5814cf7 100644 --- a/plugins/obs-outputs/rtmp-stream.c +++ b/plugins/obs-outputs/rtmp-stream.c @@ -423,13 +423,23 @@ static int send_packet(struct rtmp_stream *stream, } } - flv_packet_mux(packet, is_header ? 0 : stream->start_dts_offset, &data, - &size, is_header); + if (stream->using_metadata_multitrack && idx > 0) { + flv_additional_packet_mux( + packet, is_header ? 0 : stream->start_dts_offset, &data, + &size, is_header, idx); + } else { + flv_packet_mux(packet, is_header ? 0 : stream->start_dts_offset, + &data, &size, is_header); + } #ifdef TEST_FRAMEDROPS droptest_cap_data_rate(stream, size); #endif + if (stream->using_metadata_multitrack) { + idx = 0; + } + ret = RTMP_Write(&stream->rtmp, (char *)data, (int)size, (int)idx); bfree(data); @@ -659,6 +669,23 @@ static void *send_thread(void *data) return NULL; } +static bool send_additional_meta_data(struct rtmp_stream *stream, size_t idx, + bool *next) +{ + uint8_t *meta_data; + size_t meta_data_size; + bool success = true; + + *next = flv_additional_meta_data(stream->output, &meta_data, + &meta_data_size); + + success = RTMP_Write(&stream->rtmp, (char *)meta_data, + (int)meta_data_size, 0) >= 0; + bfree(meta_data); + + return success; +} + static bool send_meta_data(struct rtmp_stream *stream, size_t idx, bool *next) { uint8_t *meta_data; @@ -866,7 +893,13 @@ static int init_send(struct rtmp_stream *stream) os_atomic_set_bool(&stream->active, true); while (next) { - if (!send_meta_data(stream, idx++, &next)) { + bool success = + (stream->using_metadata_multitrack && idx != 0) + ? send_additional_meta_data(stream, idx, &next) + : send_meta_data(stream, idx, &next); + idx++; + + if (!success) { warn("Disconnected while attempting to connect to " "server."); set_output_error(stream); @@ -982,16 +1015,18 @@ static int try_connect(struct rtmp_stream *stream) RTMP_AddStream(&stream->rtmp, stream->key.array); - for (size_t idx = 1;; idx++) { - obs_encoder_t *encoder = - obs_output_get_audio_encoder(stream->output, idx); - const char *encoder_name; + if (!stream->using_metadata_multitrack) { + for (size_t idx = 1;; idx++) { + obs_encoder_t *encoder = obs_output_get_audio_encoder( + stream->output, idx); + const char *encoder_name; - if (!encoder) - break; + if (!encoder) + break; - encoder_name = obs_encoder_get_name(encoder); - RTMP_AddStream(&stream->rtmp, encoder_name); + encoder_name = obs_encoder_get_name(encoder); + RTMP_AddStream(&stream->rtmp, encoder_name); + } } stream->rtmp.m_outChunkSize = 4096; @@ -1052,6 +1087,8 @@ static bool init_connect(struct rtmp_stream *stream) drop_p = (int64_t)obs_data_get_int(settings, OPT_PFRAME_DROP_THRESHOLD); stream->max_shutdown_time_sec = (int)obs_data_get_int(settings, OPT_MAX_SHUTDOWN_TIME_SEC); + stream->using_metadata_multitrack = + obs_data_get_bool(settings, OPT_METADATA_MULTITRACK); obs_encoder_t *venc = obs_output_get_video_encoder(stream->output); obs_encoder_t *aenc = obs_output_get_audio_encoder(stream->output, 0); @@ -1451,6 +1488,7 @@ static void rtmp_stream_defaults(obs_data_t *defaults) obs_data_set_default_string(defaults, OPT_BIND_IP, "default"); obs_data_set_default_bool(defaults, OPT_NEWSOCKETLOOP_ENABLED, false); obs_data_set_default_bool(defaults, OPT_LOWLATENCY_ENABLED, false); + obs_data_set_default_bool(defaults, OPT_METADATA_MULTITRACK, true); } static obs_properties_t *rtmp_stream_properties(void *unused) diff --git a/plugins/obs-outputs/rtmp-stream.h b/plugins/obs-outputs/rtmp-stream.h index 2aeb78744..7c3a9ed2c 100644 --- a/plugins/obs-outputs/rtmp-stream.h +++ b/plugins/obs-outputs/rtmp-stream.h @@ -31,6 +31,7 @@ #define OPT_BIND_IP "bind_ip" #define OPT_NEWSOCKETLOOP_ENABLED "new_socket_loop_enabled" #define OPT_LOWLATENCY_ENABLED "low_latency_mode_enabled" +#define OPT_METADATA_MULTITRACK "metadata_multitrack" //#define TEST_FRAMEDROPS //#define TEST_FRAMEDROPS_WITH_BITRATE_SHORTCUTS @@ -58,6 +59,7 @@ struct rtmp_stream { pthread_mutex_t packets_mutex; struct circlebuf packets; bool sent_headers; + bool using_metadata_multitrack; bool got_first_video; int64_t start_dts_offset; diff --git a/plugins/rtmp-services/rtmp-common.c b/plugins/rtmp-services/rtmp-common.c index 88ba6d125..913439dc1 100644 --- a/plugins/rtmp-services/rtmp-common.c +++ b/plugins/rtmp-services/rtmp-common.c @@ -14,6 +14,8 @@ struct rtmp_common { char *key; char *output; + + bool supports_additional_audio_track; }; static const char *rtmp_common_getname(void *unused) @@ -25,6 +27,7 @@ static const char *rtmp_common_getname(void *unused) static json_t *open_services_file(void); static inline json_t *find_service(json_t *root, const char *name, const char **p_new_name); +static inline bool get_bool_val(json_t *service, const char *key); static inline const char *get_string_val(json_t *service, const char *key); extern void twitch_ingests_refresh(int seconds); @@ -74,6 +77,7 @@ static void rtmp_common_update(void *data, obs_data_t *settings) service->service = bstrdup(obs_data_get_string(settings, "service")); service->server = bstrdup(obs_data_get_string(settings, "server")); service->key = bstrdup(obs_data_get_string(settings, "key")); + service->supports_additional_audio_track = false; service->output = NULL; json_t *root = open_services_file(); @@ -94,6 +98,8 @@ static void rtmp_common_update(void *data, obs_data_t *settings) service->output = bstrdup(out); } + service->supports_additional_audio_track = get_bool_val( + serv, "supports_additional_audio_track"); ensure_valid_url(service, serv, settings); } } @@ -622,6 +628,12 @@ static const char *rtmp_common_key(void *data) return service->key; } +static bool supports_multitrack(void *data) +{ + struct rtmp_common *service = data; + return service->supports_additional_audio_track; +} + struct obs_service_info rtmp_common_service = { .id = "rtmp_common", .get_name = rtmp_common_getname, @@ -633,4 +645,5 @@ struct obs_service_info rtmp_common_service = { .get_key = rtmp_common_key, .apply_encoder_settings = rtmp_common_apply_settings, .get_output_type = rtmp_common_get_output_type, + .supports_multitrack = supports_multitrack, };