From c5ef7beda83cc23563a2785995c07f7c83e04fd9 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 21 Sep 2023 20:22:49 +0200 Subject: [PATCH] obs-outputs: Add initial eRTMP multitrack implementation --- plugins/obs-outputs/flv-mux.c | 115 +++++++++++++++++------ plugins/obs-outputs/flv-mux.h | 10 +- plugins/obs-outputs/rtmp-stream.c | 151 ++++++++++++++++++++---------- plugins/obs-outputs/rtmp-stream.h | 2 +- 4 files changed, 192 insertions(+), 86 deletions(-) diff --git a/plugins/obs-outputs/flv-mux.c b/plugins/obs-outputs/flv-mux.c index 541176f69..e544da85f 100644 --- a/plugins/obs-outputs/flv-mux.c +++ b/plugins/obs-outputs/flv-mux.c @@ -32,6 +32,7 @@ #define AUDIODATA_AAC 10.0 #define VIDEO_FRAMETYPE_OFFSET 4 + enum video_frametype_t { FT_KEY = 1 << VIDEO_FRAMETYPE_OFFSET, FT_INTER = 2 << VIDEO_FRAMETYPE_OFFSET, @@ -43,10 +44,16 @@ enum packet_type_t { PACKETTYPE_SEQ_START = 0, PACKETTYPE_FRAMES = 1, PACKETTYPE_SEQ_END = 2, -#ifdef ENABLE_HEVC PACKETTYPE_FRAMESX = 3, -#endif - PACKETTYPE_METADATA = 4 + PACKETTYPE_METADATA = 4, + PACKETTYPE_MPEG2TS_SEQ_START = 5, + PACKETTYPE_MULTITRACK = 6 +}; + +enum multitrack_type_t { + MULTITRACKTYPE_ONE_TRACK = 0x00, + MULTITRACKTYPE_MANY_TRACKS = 0x10, + MULTITRACKTYPE_MANY_TRACKS_MANY_CODECS = 0x20, }; enum datatype_t { @@ -76,7 +83,11 @@ static void s_w4cc(struct serializer *s, enum video_id_t id) assert(0); #endif case CODEC_H264: - assert(0); + s_w8(s, 'a'); + s_w8(s, 'v'); + s_w8(s, 'c'); + s_w8(s, '1'); + break; } } @@ -363,7 +374,8 @@ void flv_packet_mux(struct encoder_packet *packet, int32_t dts_offset, // Y2023 spec void flv_packet_ex(struct encoder_packet *packet, enum video_id_t codec_id, - int32_t dts_offset, uint8_t **output, size_t *size, int type) + int32_t dts_offset, uint8_t **output, size_t *size, int type, + size_t idx) { struct array_output_data data; struct serializer s; @@ -373,30 +385,45 @@ void flv_packet_ex(struct encoder_packet *packet, enum video_id_t codec_id, int32_t time_ms = get_ms_time(packet, packet->dts) - dts_offset; + bool is_multitrack = idx > 0; + // packet head - int header_metadata_size = 5; -#ifdef ENABLE_HEVC + int header_metadata_size = 5; // w8+w4cc // 3 extra bytes for composition time offset - if (codec_id == CODEC_HEVC && type == PACKETTYPE_FRAMES) { - header_metadata_size = 8; + if ((codec_id == CODEC_H264 || codec_id == CODEC_HEVC) && + type == PACKETTYPE_FRAMES) { + header_metadata_size += 3; // w24 } -#endif + if (is_multitrack) + header_metadata_size += 2; // w8+w8 + s_w8(&s, RTMP_PACKET_TYPE_VIDEO); s_wb24(&s, (uint32_t)packet->size + header_metadata_size); s_wtimestamp(&s, time_ms); s_wb24(&s, 0); // always 0 - // packet ext header - s_w8(&s, - FRAME_HEADER_EX | type | (packet->keyframe ? FT_KEY : FT_INTER)); - s_w4cc(&s, codec_id); + uint8_t frame_type = packet->keyframe ? FT_KEY : FT_INTER; -#ifdef ENABLE_HEVC - // hevc composition time offset - if (codec_id == CODEC_HEVC && type == PACKETTYPE_FRAMES) { + /* + * We only explicitly emit trackIds iff idx > 0. + * The default trackId is 0. + */ + if (is_multitrack) { + s_w8(&s, FRAME_HEADER_EX | PACKETTYPE_MULTITRACK | frame_type); + s_w8(&s, MULTITRACKTYPE_ONE_TRACK | type); + s_w4cc(&s, codec_id); + // trackId + s_w8(&s, (uint8_t)idx); + } else { + s_w8(&s, FRAME_HEADER_EX | type | frame_type); + s_w4cc(&s, codec_id); + } + + // H.264/HEVC composition time offset + if ((codec_id == CODEC_H264 || codec_id == CODEC_HEVC) && + type == PACKETTYPE_FRAMES) { s_wb24(&s, get_ms_time(packet, packet->pts - packet->dts)); } -#endif // packet data s_write(&s, packet->data, packet->size); @@ -409,34 +436,37 @@ void flv_packet_ex(struct encoder_packet *packet, enum video_id_t codec_id, } void flv_packet_start(struct encoder_packet *packet, enum video_id_t codec, - uint8_t **output, size_t *size) + uint8_t **output, size_t *size, size_t idx) { - flv_packet_ex(packet, codec, 0, output, size, PACKETTYPE_SEQ_START); + flv_packet_ex(packet, codec, 0, output, size, PACKETTYPE_SEQ_START, + idx); } void flv_packet_frames(struct encoder_packet *packet, enum video_id_t codec, - int32_t dts_offset, uint8_t **output, size_t *size) + int32_t dts_offset, uint8_t **output, size_t *size, + size_t idx) { int packet_type = PACKETTYPE_FRAMES; -#ifdef ENABLE_HEVC // PACKETTYPE_FRAMESX is an optimization to avoid sending composition // time offsets of 0. See Enhanced RTMP spec. - if (codec == CODEC_HEVC && packet->dts == packet->pts) + if ((codec == CODEC_H264 || codec == CODEC_HEVC) && + packet->dts == packet->pts) packet_type = PACKETTYPE_FRAMESX; -#endif - flv_packet_ex(packet, codec, dts_offset, output, size, packet_type); + flv_packet_ex(packet, codec, dts_offset, output, size, packet_type, + idx); } void flv_packet_end(struct encoder_packet *packet, enum video_id_t codec, - uint8_t **output, size_t *size) + uint8_t **output, size_t *size, size_t idx) { - flv_packet_ex(packet, codec, 0, output, size, PACKETTYPE_SEQ_END); + flv_packet_ex(packet, codec, 0, output, size, PACKETTYPE_SEQ_END, idx); } void flv_packet_metadata(enum video_id_t codec_id, uint8_t **output, size_t *size, int bits_per_raw_sample, uint8_t color_primaries, int color_trc, - int color_space, int min_luminance, int max_luminance) + int color_space, int min_luminance, int max_luminance, + size_t idx) { // metadata array struct array_output_data data; @@ -500,16 +530,39 @@ void flv_packet_metadata(enum video_id_t codec_id, uint8_t **output, s_w8(&s, DATA_TYPE_OBJECT_END); } + bool is_multitrack = idx > 0; + // packet head + // w8+w4cc + int header_metadata_size = 5; + if (is_multitrack) { + // w8+w8 + header_metadata_size += 2; + } + s_w8(&s, RTMP_PACKET_TYPE_VIDEO); - s_wb24(&s, (uint32_t)metadata.bytes.num + 5); // 5 = (w8+w4cc) + s_wb24(&s, (uint32_t)metadata.bytes.num + header_metadata_size); s_wtimestamp(&s, 0); s_wb24(&s, 0); // always 0 // packet ext header // these are the 5 extra bytes mentioned above - s_w8(&s, FRAME_HEADER_EX | PACKETTYPE_METADATA); - s_w4cc(&s, codec_id); + s_w8(&s, FRAME_HEADER_EX | (is_multitrack ? PACKETTYPE_MULTITRACK + : PACKETTYPE_METADATA)); + + /* + * We only add explicitly emit trackIds iff idx > 0. + * The default trackId is 0. + */ + if (is_multitrack) { + s_w8(&s, MULTITRACKTYPE_ONE_TRACK | PACKETTYPE_METADATA); + s_w4cc(&s, codec_id); + // trackId + s_w8(&s, (uint8_t)idx); + } else { + s_w4cc(&s, codec_id); + } + // packet data s_write(&s, metadata.bytes.array, metadata.bytes.num); array_output_serializer_free(&metadata); // must be freed diff --git a/plugins/obs-outputs/flv-mux.h b/plugins/obs-outputs/flv-mux.h index f1cd884f6..032cc8876 100644 --- a/plugins/obs-outputs/flv-mux.h +++ b/plugins/obs-outputs/flv-mux.h @@ -22,7 +22,7 @@ #define MILLISECOND_DEN 1000 enum video_id_t { - CODEC_H264 = 1, // legacy + CODEC_H264 = 1, // legacy & Y2023 spec CODEC_AV1, // Y2023 spec CODEC_HEVC, }; @@ -60,14 +60,14 @@ extern void flv_additional_packet_mux(struct encoder_packet *packet, // Y2023 spec extern void flv_packet_start(struct encoder_packet *packet, enum video_id_t codec, uint8_t **output, - size_t *size); + size_t *size, size_t idx); extern void flv_packet_frames(struct encoder_packet *packet, enum video_id_t codec, int32_t dts_offset, - uint8_t **output, size_t *size); + uint8_t **output, size_t *size, size_t idx); extern void flv_packet_end(struct encoder_packet *packet, enum video_id_t codec, - uint8_t **output, size_t *size); + uint8_t **output, size_t *size, size_t idx); extern void flv_packet_metadata(enum video_id_t codec, uint8_t **output, size_t *size, int bits_per_raw_sample, uint8_t color_primaries, int color_trc, int color_space, int min_luminance, - int max_luminance); + int max_luminance, size_t idx); diff --git a/plugins/obs-outputs/rtmp-stream.c b/plugins/obs-outputs/rtmp-stream.c index f1c090822..d3075b64b 100644 --- a/plugins/obs-outputs/rtmp-stream.c +++ b/plugins/obs-outputs/rtmp-stream.c @@ -444,7 +444,7 @@ static int send_packet(struct rtmp_stream *stream, static int send_packet_ex(struct rtmp_stream *stream, struct encoder_packet *packet, bool is_header, - bool is_footer) + bool is_footer, size_t idx) { uint8_t *data; size_t size = 0; @@ -454,12 +454,14 @@ static int send_packet_ex(struct rtmp_stream *stream, return -1; if (is_header) { - flv_packet_start(packet, stream->video_codec, &data, &size); + flv_packet_start(packet, stream->video_codec[idx], &data, &size, + idx); } else if (is_footer) { - flv_packet_end(packet, stream->video_codec, &data, &size); + flv_packet_end(packet, stream->video_codec[idx], &data, &size, + idx); } else { - flv_packet_frames(packet, stream->video_codec, - stream->start_dts_offset, &data, &size); + flv_packet_frames(packet, stream->video_codec[idx], + stream->start_dts_offset, &data, &size, idx); } #ifdef TEST_FRAMEDROPS @@ -669,8 +671,11 @@ static void *send_thread(void *data) int sent; if (packet.type == OBS_ENCODER_VIDEO && - stream->video_codec != CODEC_H264) { - sent = send_packet_ex(stream, &packet, false, false); + (stream->video_codec[packet.track_idx] != CODEC_H264 || + (stream->video_codec[packet.track_idx] == CODEC_H264 && + packet.track_idx != 0))) { + sent = send_packet_ex(stream, &packet, false, false, + packet.track_idx); } else { sent = send_packet(stream, &packet, false, packet.track_idx); @@ -792,10 +797,10 @@ static bool send_audio_header(struct rtmp_stream *stream, size_t idx, return send_packet(stream, &packet, true, idx) >= 0; } -static bool send_video_header(struct rtmp_stream *stream) +static bool send_video_header(struct rtmp_stream *stream, size_t idx) { obs_output_t *context = stream->output; - obs_encoder_t *vencoder = obs_output_get_video_encoder(context); + obs_encoder_t *vencoder = obs_output_get_video_encoder2(context, idx); uint8_t *header; size_t size; @@ -803,35 +808,61 @@ static bool send_video_header(struct rtmp_stream *stream) .timebase_den = 1, .keyframe = true}; + if (!vencoder) + return false; + if (!obs_encoder_get_extra_data(vencoder, &header, &size)) return false; - switch (stream->video_codec) { + switch (stream->video_codec[idx]) { case CODEC_H264: packet.size = obs_parse_avc_header(&packet.data, header, size); - return send_packet(stream, &packet, true, 0) >= 0; + // Always send H.264 on track 0 as old style for compatibility. + if (idx == 0) { + return send_packet(stream, &packet, true, idx) >= 0; + } else { + return send_packet_ex(stream, &packet, true, false, + idx) >= 0; + } case CODEC_HEVC: #ifdef ENABLE_HEVC packet.size = obs_parse_hevc_header(&packet.data, header, size); - return send_packet_ex(stream, &packet, true, 0) >= 0; + return send_packet_ex(stream, &packet, true, false, idx) >= 0; #else return false; #endif case CODEC_AV1: packet.size = obs_parse_av1_header(&packet.data, header, size); - return send_packet_ex(stream, &packet, true, 0) >= 0; + return send_packet_ex(stream, &packet, true, false, idx) >= 0; } return false; } -static bool send_video_metadata(struct rtmp_stream *stream) +// only returns false if there's an error, not if no metadata needs to be sent +static bool send_video_metadata(struct rtmp_stream *stream, size_t idx) { + // send metadata only if HDR + obs_encoder_t *encoder = + obs_output_get_video_encoder2(stream->output, idx); + if (!encoder) + return false; + + video_t *video = obs_encoder_video(encoder); + if (!video) + return false; + + const struct video_output_info *info = video_output_get_info(video); + enum video_colorspace colorspace = info->colorspace; + if (!(colorspace == VIDEO_CS_2100_PQ || + colorspace == VIDEO_CS_2100_HLG)) + return true; + if (handle_socket_read(stream)) - return -1; + return false; // Y2023 spec - if (stream->video_codec != CODEC_H264) { + if (stream->video_codec[idx] != CODEC_H264) { uint8_t *data; size_t size; @@ -892,9 +923,9 @@ static bool send_video_metadata(struct rtmp_stream *stream) max_luminance = (int)obs_get_video_hdr_nominal_peak_level(); - flv_packet_metadata(stream->video_codec, &data, &size, + flv_packet_metadata(stream->video_codec[idx], &data, &size, bits_per_raw_sample, pri, trc, spc, 0, - max_luminance); + max_luminance, idx); int ret = RTMP_Write(&stream->rtmp, (char *)data, (int)size, 0); bfree(data); @@ -906,14 +937,14 @@ static bool send_video_metadata(struct rtmp_stream *stream) return true; } -static bool send_video_footer(struct rtmp_stream *stream) +static bool send_video_footer(struct rtmp_stream *stream, size_t idx) { struct encoder_packet packet = {.type = OBS_ENCODER_VIDEO, .timebase_den = 1, .keyframe = false}; packet.size = 0; - return send_packet_ex(stream, &packet, 0, true) >= 0; + return send_packet_ex(stream, &packet, false, true, idx) >= 0; } static inline bool send_headers(struct rtmp_stream *stream) @@ -925,16 +956,16 @@ static inline bool send_headers(struct rtmp_stream *stream) if (!send_audio_header(stream, i++, &next)) return false; - // send metadata only if HDR - video_t *video = obs_get_video(); - const struct video_output_info *info = video_output_get_info(video); - enum video_colorspace colorspace = info->colorspace; - if (colorspace == VIDEO_CS_2100_PQ || colorspace == VIDEO_CS_2100_HLG) - if (!send_video_metadata(stream)) // Y2023 spec - return false; + for (size_t j = 0; j < MAX_OUTPUT_VIDEO_ENCODERS; j++) { + obs_output_t *enc = + obs_output_get_video_encoder2(stream->output, j); + if (!enc) + continue; - if (!send_video_header(stream)) - return false; + if (!send_video_metadata(stream, j) || + !send_video_header(stream, j)) + return false; + } while (next) { if (!send_audio_header(stream, i++, &next)) @@ -946,11 +977,20 @@ static inline bool send_headers(struct rtmp_stream *stream) static inline bool send_footers(struct rtmp_stream *stream) { - if (stream->video_codec == CODEC_H264) - return false; + for (size_t i = 0; i < MAX_OUTPUT_VIDEO_ENCODERS; i++) { + obs_encoder_t *encoder = + obs_output_get_video_encoder2(stream->output, i); + if (!encoder) + continue; - // Y2023 spec - return send_video_footer(stream); + if (i == 0 && stream->video_codec[i] == CODEC_H264) + continue; + + if (!send_video_footer(stream, i)) + return false; + } + + return true; } static inline bool reset_semaphore(struct rtmp_stream *stream) @@ -997,8 +1037,12 @@ static int init_send(struct rtmp_stream *stream) int total_bitrate = 0; - obs_encoder_t *vencoder = obs_output_get_video_encoder(context); - if (vencoder) { + for (size_t i = 0; i < MAX_OUTPUT_VIDEO_ENCODERS; i++) { + obs_encoder_t *vencoder = + obs_output_get_video_encoder2(context, i); + if (!vencoder) + continue; + obs_data_t *params = obs_encoder_get_settings(vencoder); if (params) { int bitrate = @@ -1284,9 +1328,15 @@ static bool init_connect(struct rtmp_stream *stream) obs_encoder_t *aenc = obs_output_get_audio_encoder(stream->output, 0); obs_data_t *vsettings = obs_encoder_get_settings(venc); obs_data_t *asettings = obs_encoder_get_settings(aenc); + for (size_t i = 0; i < MAX_OUTPUT_VIDEO_ENCODERS; i++) { + obs_encoder_t *enc = + obs_output_get_video_encoder2(stream->output, i); - const char *codec = obs_encoder_get_codec(venc); - stream->video_codec = to_video_type(codec); + if (enc) { + const char *codec = obs_encoder_get_codec(enc); + stream->video_codec[i] = to_video_type(codec); + } + } deque_free(&stream->dbr_frames); stream->audio_bitrate = (long)obs_data_get_int(asettings, "bitrate"); @@ -1371,17 +1421,20 @@ static void *connect_thread(void *data) } // HDR streaming disabled for AV1 - if (stream->video_codec != CODEC_H264 && - stream->video_codec != CODEC_HEVC) { - video_t *video = obs_get_video(); - const struct video_output_info *info = - video_output_get_info(video); + for (size_t i = 0; i < MAX_OUTPUT_VIDEO_ENCODERS; i++) { + if (stream->video_codec[i] && + stream->video_codec[i] != CODEC_H264 && + stream->video_codec[i] != CODEC_HEVC) { + video_t *video = obs_get_video(); + const struct video_output_info *info = + video_output_get_info(video); - if (info->colorspace == VIDEO_CS_2100_HLG || - info->colorspace == VIDEO_CS_2100_PQ) { - obs_output_signal_stop(stream->output, - OBS_OUTPUT_HDR_DISABLED); - return NULL; + if (info->colorspace == VIDEO_CS_2100_HLG || + info->colorspace == VIDEO_CS_2100_PQ) { + obs_output_signal_stop(stream->output, + OBS_OUTPUT_HDR_DISABLED); + return NULL; + } } } @@ -1693,7 +1746,7 @@ static void rtmp_stream_data(void *data, struct encoder_packet *packet) stream->got_first_video = true; } - switch (stream->video_codec) { + switch (stream->video_codec[packet->track_idx]) { case CODEC_H264: obs_parse_avc_packet(&new_packet, packet); break; @@ -1821,7 +1874,7 @@ static int rtmp_stream_connect_time(void *data) struct obs_output_info rtmp_output_info = { .id = "rtmp_output", .flags = OBS_OUTPUT_AV | OBS_OUTPUT_ENCODED | OBS_OUTPUT_SERVICE | - OBS_OUTPUT_MULTI_TRACK, + OBS_OUTPUT_MULTI_TRACK_AV, #ifdef NO_CRYPTO .protocols = "RTMP", #else diff --git a/plugins/obs-outputs/rtmp-stream.h b/plugins/obs-outputs/rtmp-stream.h index 29912e501..6444faeab 100644 --- a/plugins/obs-outputs/rtmp-stream.h +++ b/plugins/obs-outputs/rtmp-stream.h @@ -114,7 +114,7 @@ struct rtmp_stream { long dbr_inc_bitrate; bool dbr_enabled; - enum video_id_t video_codec; + enum video_id_t video_codec[MAX_OUTPUT_VIDEO_ENCODERS]; RTMP rtmp;