obs-outputs: Add initial eRTMP multitrack implementation

plugins/obs-outputs/flv-mux.c

@@ -32,6 +32,7 @@
 #define AUDIODATA_AAC 10.0
 
 #define VIDEO_FRAMETYPE_OFFSET 4
+
 enum video_frametype_t {
 	FT_KEY = 1 << VIDEO_FRAMETYPE_OFFSET,
 	FT_INTER = 2 << VIDEO_FRAMETYPE_OFFSET,
@@ -43,10 +44,16 @@ enum packet_type_t {
 	PACKETTYPE_SEQ_START = 0,
 	PACKETTYPE_FRAMES = 1,
 	PACKETTYPE_SEQ_END = 2,
-#ifdef ENABLE_HEVC
 	PACKETTYPE_FRAMESX = 3,
-#endif
-	PACKETTYPE_METADATA = 4
+	PACKETTYPE_METADATA = 4,
+	PACKETTYPE_MPEG2TS_SEQ_START = 5,
+	PACKETTYPE_MULTITRACK = 6
+};
+
+enum multitrack_type_t {
+	MULTITRACKTYPE_ONE_TRACK = 0x00,
+	MULTITRACKTYPE_MANY_TRACKS = 0x10,
+	MULTITRACKTYPE_MANY_TRACKS_MANY_CODECS = 0x20,
 };
 
 enum datatype_t {
@@ -76,7 +83,11 @@ static void s_w4cc(struct serializer *s, enum video_id_t id)
 		assert(0);
 #endif
 	case CODEC_H264:
-		assert(0);
+		s_w8(s, 'a');
+		s_w8(s, 'v');
+		s_w8(s, 'c');
+		s_w8(s, '1');
+		break;
 	}
 }
 
@@ -363,7 +374,8 @@ void flv_packet_mux(struct encoder_packet *packet, int32_t dts_offset,
 
 // Y2023 spec
 void flv_packet_ex(struct encoder_packet *packet, enum video_id_t codec_id,
-		   int32_t dts_offset, uint8_t **output, size_t *size, int type)
+		   int32_t dts_offset, uint8_t **output, size_t *size, int type,
+		   size_t idx)
 {
 	struct array_output_data data;
 	struct serializer s;
@@ -373,30 +385,45 @@ void flv_packet_ex(struct encoder_packet *packet, enum video_id_t codec_id,
 
 	int32_t time_ms = get_ms_time(packet, packet->dts) - dts_offset;
 
+	bool is_multitrack = idx > 0;
+
 	// packet head
-	int header_metadata_size = 5;
-#ifdef ENABLE_HEVC
+	int header_metadata_size = 5; // w8+w4cc
 	// 3 extra bytes for composition time offset
-	if (codec_id == CODEC_HEVC && type == PACKETTYPE_FRAMES) {
-		header_metadata_size = 8;
+	if ((codec_id == CODEC_H264 || codec_id == CODEC_HEVC) &&
+	    type == PACKETTYPE_FRAMES) {
+		header_metadata_size += 3; // w24
 	}
-#endif
+	if (is_multitrack)
+		header_metadata_size += 2; // w8+w8
+
 	s_w8(&s, RTMP_PACKET_TYPE_VIDEO);
 	s_wb24(&s, (uint32_t)packet->size + header_metadata_size);
 	s_wtimestamp(&s, time_ms);
 	s_wb24(&s, 0); // always 0
 
-	// packet ext header
-	s_w8(&s,
-	     FRAME_HEADER_EX | type | (packet->keyframe ? FT_KEY : FT_INTER));
-	s_w4cc(&s, codec_id);
+	uint8_t frame_type = packet->keyframe ? FT_KEY : FT_INTER;
 
-#ifdef ENABLE_HEVC
-	// hevc composition time offset
-	if (codec_id == CODEC_HEVC && type == PACKETTYPE_FRAMES) {
+	/*
+	 * We only explicitly emit trackIds iff idx > 0.
+	 * The default trackId is 0.
+	 */
+	if (is_multitrack) {
+		s_w8(&s, FRAME_HEADER_EX | PACKETTYPE_MULTITRACK | frame_type);
+		s_w8(&s, MULTITRACKTYPE_ONE_TRACK | type);
+		s_w4cc(&s, codec_id);
+		// trackId
+		s_w8(&s, (uint8_t)idx);
+	} else {
+		s_w8(&s, FRAME_HEADER_EX | type | frame_type);
+		s_w4cc(&s, codec_id);
+	}
+
+	// H.264/HEVC composition time offset
+	if ((codec_id == CODEC_H264 || codec_id == CODEC_HEVC) &&
+	    type == PACKETTYPE_FRAMES) {
 		s_wb24(&s, get_ms_time(packet, packet->pts - packet->dts));
 	}
-#endif
 
 	// packet data
 	s_write(&s, packet->data, packet->size);
@@ -409,34 +436,37 @@ void flv_packet_ex(struct encoder_packet *packet, enum video_id_t codec_id,
 }
 
 void flv_packet_start(struct encoder_packet *packet, enum video_id_t codec,
-		      uint8_t **output, size_t *size)
+		      uint8_t **output, size_t *size, size_t idx)
 {
-	flv_packet_ex(packet, codec, 0, output, size, PACKETTYPE_SEQ_START);
+	flv_packet_ex(packet, codec, 0, output, size, PACKETTYPE_SEQ_START,
+		      idx);
 }
 
 void flv_packet_frames(struct encoder_packet *packet, enum video_id_t codec,
-		       int32_t dts_offset, uint8_t **output, size_t *size)
+		       int32_t dts_offset, uint8_t **output, size_t *size,
+		       size_t idx)
 {
 	int packet_type = PACKETTYPE_FRAMES;
-#ifdef ENABLE_HEVC
 	// PACKETTYPE_FRAMESX is an optimization to avoid sending composition
 	// time offsets of 0. See Enhanced RTMP spec.
-	if (codec == CODEC_HEVC && packet->dts == packet->pts)
+	if ((codec == CODEC_H264 || codec == CODEC_HEVC) &&
+	    packet->dts == packet->pts)
 		packet_type = PACKETTYPE_FRAMESX;
-#endif
-	flv_packet_ex(packet, codec, dts_offset, output, size, packet_type);
+	flv_packet_ex(packet, codec, dts_offset, output, size, packet_type,
+		      idx);
 }
 
 void flv_packet_end(struct encoder_packet *packet, enum video_id_t codec,
-		    uint8_t **output, size_t *size)
+		    uint8_t **output, size_t *size, size_t idx)
 {
-	flv_packet_ex(packet, codec, 0, output, size, PACKETTYPE_SEQ_END);
+	flv_packet_ex(packet, codec, 0, output, size, PACKETTYPE_SEQ_END, idx);
 }
 
 void flv_packet_metadata(enum video_id_t codec_id, uint8_t **output,
 			 size_t *size, int bits_per_raw_sample,
 			 uint8_t color_primaries, int color_trc,
-			 int color_space, int min_luminance, int max_luminance)
+			 int color_space, int min_luminance, int max_luminance,
+			 size_t idx)
 {
 	// metadata array
 	struct array_output_data data;
@@ -500,16 +530,39 @@ void flv_packet_metadata(enum video_id_t codec_id, uint8_t **output,
 		s_w8(&s, DATA_TYPE_OBJECT_END);
 	}
 
+	bool is_multitrack = idx > 0;
+
 	// packet head
+	// w8+w4cc
+	int header_metadata_size = 5;
+	if (is_multitrack) {
+		// w8+w8
+		header_metadata_size += 2;
+	}
+
 	s_w8(&s, RTMP_PACKET_TYPE_VIDEO);
-	s_wb24(&s, (uint32_t)metadata.bytes.num + 5); // 5 = (w8+w4cc)
+	s_wb24(&s, (uint32_t)metadata.bytes.num + header_metadata_size);
 	s_wtimestamp(&s, 0);
 	s_wb24(&s, 0); // always 0
 
 	// packet ext header
 	// these are the 5 extra bytes mentioned above
-	s_w8(&s, FRAME_HEADER_EX | PACKETTYPE_METADATA);
-	s_w4cc(&s, codec_id);
+	s_w8(&s, FRAME_HEADER_EX | (is_multitrack ? PACKETTYPE_MULTITRACK
+						  : PACKETTYPE_METADATA));
+
+	/*
+	 * We only add explicitly emit trackIds iff idx > 0.
+	 * The default trackId is 0.
+	 */
+	if (is_multitrack) {
+		s_w8(&s, MULTITRACKTYPE_ONE_TRACK | PACKETTYPE_METADATA);
+		s_w4cc(&s, codec_id);
+		// trackId
+		s_w8(&s, (uint8_t)idx);
+	} else {
+		s_w4cc(&s, codec_id);
+	}
+
 	// packet data
 	s_write(&s, metadata.bytes.array, metadata.bytes.num);
 	array_output_serializer_free(&metadata); // must be freed

plugins/obs-outputs/flv-mux.h

@@ -22,7 +22,7 @@
 #define MILLISECOND_DEN 1000
 
 enum video_id_t {
-	CODEC_H264 = 1, // legacy
+	CODEC_H264 = 1, // legacy & Y2023 spec
 	CODEC_AV1,      // Y2023 spec
 	CODEC_HEVC,
 };
@@ -60,14 +60,14 @@ extern void flv_additional_packet_mux(struct encoder_packet *packet,
 // Y2023 spec
 extern void flv_packet_start(struct encoder_packet *packet,
 			     enum video_id_t codec, uint8_t **output,
-			     size_t *size);
+			     size_t *size, size_t idx);
 extern void flv_packet_frames(struct encoder_packet *packet,
 			      enum video_id_t codec, int32_t dts_offset,
-			      uint8_t **output, size_t *size);
+			      uint8_t **output, size_t *size, size_t idx);
 extern void flv_packet_end(struct encoder_packet *packet, enum video_id_t codec,
-			   uint8_t **output, size_t *size);
+			   uint8_t **output, size_t *size, size_t idx);
 extern void flv_packet_metadata(enum video_id_t codec, uint8_t **output,
 				size_t *size, int bits_per_raw_sample,
 				uint8_t color_primaries, int color_trc,
 				int color_space, int min_luminance,
-				int max_luminance);
+				int max_luminance, size_t idx);

plugins/obs-outputs/rtmp-stream.c

@@ -444,7 +444,7 @@ static int send_packet(struct rtmp_stream *stream,
 
 static int send_packet_ex(struct rtmp_stream *stream,
 			  struct encoder_packet *packet, bool is_header,
-			  bool is_footer)
+			  bool is_footer, size_t idx)
 {
 	uint8_t *data;
 	size_t size = 0;
@@ -454,12 +454,14 @@ static int send_packet_ex(struct rtmp_stream *stream,
 		return -1;
 
 	if (is_header) {
-		flv_packet_start(packet, stream->video_codec, &data, &size);
+		flv_packet_start(packet, stream->video_codec[idx], &data, &size,
+				 idx);
 	} else if (is_footer) {
-		flv_packet_end(packet, stream->video_codec, &data, &size);
+		flv_packet_end(packet, stream->video_codec[idx], &data, &size,
+			       idx);
 	} else {
-		flv_packet_frames(packet, stream->video_codec,
-				  stream->start_dts_offset, &data, &size);
+		flv_packet_frames(packet, stream->video_codec[idx],
+				  stream->start_dts_offset, &data, &size, idx);
 	}
 
 #ifdef TEST_FRAMEDROPS
@@ -669,8 +671,11 @@ static void *send_thread(void *data)
 
 		int sent;
 		if (packet.type == OBS_ENCODER_VIDEO &&
-		    stream->video_codec != CODEC_H264) {
-			sent = send_packet_ex(stream, &packet, false, false);
+		    (stream->video_codec[packet.track_idx] != CODEC_H264 ||
+		     (stream->video_codec[packet.track_idx] == CODEC_H264 &&
+		      packet.track_idx != 0))) {
+			sent = send_packet_ex(stream, &packet, false, false,
+					      packet.track_idx);
 		} else {
 			sent = send_packet(stream, &packet, false,
 					   packet.track_idx);
@@ -792,10 +797,10 @@ static bool send_audio_header(struct rtmp_stream *stream, size_t idx,
 	return send_packet(stream, &packet, true, idx) >= 0;
 }
 
-static bool send_video_header(struct rtmp_stream *stream)
+static bool send_video_header(struct rtmp_stream *stream, size_t idx)
 {
 	obs_output_t *context = stream->output;
-	obs_encoder_t *vencoder = obs_output_get_video_encoder(context);
+	obs_encoder_t *vencoder = obs_output_get_video_encoder2(context, idx);
 	uint8_t *header;
 	size_t size;
 
@@ -803,35 +808,61 @@ static bool send_video_header(struct rtmp_stream *stream)
 					.timebase_den = 1,
 					.keyframe = true};
 
+	if (!vencoder)
+		return false;
+
 	if (!obs_encoder_get_extra_data(vencoder, &header, &size))
 		return false;
 
-	switch (stream->video_codec) {
+	switch (stream->video_codec[idx]) {
 	case CODEC_H264:
 		packet.size = obs_parse_avc_header(&packet.data, header, size);
-		return send_packet(stream, &packet, true, 0) >= 0;
+		// Always send H.264 on track 0 as old style for compatibility.
+		if (idx == 0) {
+			return send_packet(stream, &packet, true, idx) >= 0;
+		} else {
+			return send_packet_ex(stream, &packet, true, false,
+					      idx) >= 0;
+		}
 	case CODEC_HEVC:
 #ifdef ENABLE_HEVC
 		packet.size = obs_parse_hevc_header(&packet.data, header, size);
-		return send_packet_ex(stream, &packet, true, 0) >= 0;
+		return send_packet_ex(stream, &packet, true, false, idx) >= 0;
 #else
 		return false;
 #endif
 	case CODEC_AV1:
 		packet.size = obs_parse_av1_header(&packet.data, header, size);
-		return send_packet_ex(stream, &packet, true, 0) >= 0;
+		return send_packet_ex(stream, &packet, true, false, idx) >= 0;
 	}
 
 	return false;
 }
 
-static bool send_video_metadata(struct rtmp_stream *stream)
+// only returns false if there's an error, not if no metadata needs to be sent
+static bool send_video_metadata(struct rtmp_stream *stream, size_t idx)
 {
+	// send metadata only if HDR
+	obs_encoder_t *encoder =
+		obs_output_get_video_encoder2(stream->output, idx);
+	if (!encoder)
+		return false;
+
+	video_t *video = obs_encoder_video(encoder);
+	if (!video)
+		return false;
+
+	const struct video_output_info *info = video_output_get_info(video);
+	enum video_colorspace colorspace = info->colorspace;
+	if (!(colorspace == VIDEO_CS_2100_PQ ||
+	      colorspace == VIDEO_CS_2100_HLG))
+		return true;
+
 	if (handle_socket_read(stream))
-		return -1;
+		return false;
 
 	// Y2023 spec
-	if (stream->video_codec != CODEC_H264) {
+	if (stream->video_codec[idx] != CODEC_H264) {
 		uint8_t *data;
 		size_t size;
 
@@ -892,9 +923,9 @@ static bool send_video_metadata(struct rtmp_stream *stream)
 			max_luminance =
 				(int)obs_get_video_hdr_nominal_peak_level();
 
-		flv_packet_metadata(stream->video_codec, &data, &size,
+		flv_packet_metadata(stream->video_codec[idx], &data, &size,
 				    bits_per_raw_sample, pri, trc, spc, 0,
-				    max_luminance);
+				    max_luminance, idx);
 
 		int ret = RTMP_Write(&stream->rtmp, (char *)data, (int)size, 0);
 		bfree(data);
@@ -906,14 +937,14 @@ static bool send_video_metadata(struct rtmp_stream *stream)
 	return true;
 }
 
-static bool send_video_footer(struct rtmp_stream *stream)
+static bool send_video_footer(struct rtmp_stream *stream, size_t idx)
 {
 	struct encoder_packet packet = {.type = OBS_ENCODER_VIDEO,
 					.timebase_den = 1,
 					.keyframe = false};
 	packet.size = 0;
 
-	return send_packet_ex(stream, &packet, 0, true) >= 0;
+	return send_packet_ex(stream, &packet, false, true, idx) >= 0;
 }
 
 static inline bool send_headers(struct rtmp_stream *stream)
@@ -925,16 +956,16 @@ static inline bool send_headers(struct rtmp_stream *stream)
 	if (!send_audio_header(stream, i++, &next))
 		return false;
 
-	// send metadata only if HDR
-	video_t *video = obs_get_video();
-	const struct video_output_info *info = video_output_get_info(video);
-	enum video_colorspace colorspace = info->colorspace;
-	if (colorspace == VIDEO_CS_2100_PQ || colorspace == VIDEO_CS_2100_HLG)
-		if (!send_video_metadata(stream)) // Y2023 spec
-			return false;
+	for (size_t j = 0; j < MAX_OUTPUT_VIDEO_ENCODERS; j++) {
+		obs_output_t *enc =
+			obs_output_get_video_encoder2(stream->output, j);
+		if (!enc)
+			continue;
 
-	if (!send_video_header(stream))
-		return false;
+		if (!send_video_metadata(stream, j) ||
+		    !send_video_header(stream, j))
+			return false;
+	}
 
 	while (next) {
 		if (!send_audio_header(stream, i++, &next))
@@ -946,11 +977,20 @@ static inline bool send_headers(struct rtmp_stream *stream)
 
 static inline bool send_footers(struct rtmp_stream *stream)
 {
-	if (stream->video_codec == CODEC_H264)
-		return false;
+	for (size_t i = 0; i < MAX_OUTPUT_VIDEO_ENCODERS; i++) {
+		obs_encoder_t *encoder =
+			obs_output_get_video_encoder2(stream->output, i);
+		if (!encoder)
+			continue;
 
-	// Y2023 spec
-	return send_video_footer(stream);
+		if (i == 0 && stream->video_codec[i] == CODEC_H264)
+			continue;
+
+		if (!send_video_footer(stream, i))
+			return false;
+	}
+
+	return true;
 }
 
 static inline bool reset_semaphore(struct rtmp_stream *stream)
@@ -997,8 +1037,12 @@ static int init_send(struct rtmp_stream *stream)
 
 		int total_bitrate = 0;
 
-		obs_encoder_t *vencoder = obs_output_get_video_encoder(context);
-		if (vencoder) {
+		for (size_t i = 0; i < MAX_OUTPUT_VIDEO_ENCODERS; i++) {
+			obs_encoder_t *vencoder =
+				obs_output_get_video_encoder2(context, i);
+			if (!vencoder)
+				continue;
+
 			obs_data_t *params = obs_encoder_get_settings(vencoder);
 			if (params) {
 				int bitrate =
@@ -1284,9 +1328,15 @@ static bool init_connect(struct rtmp_stream *stream)
 	obs_encoder_t *aenc = obs_output_get_audio_encoder(stream->output, 0);
 	obs_data_t *vsettings = obs_encoder_get_settings(venc);
 	obs_data_t *asettings = obs_encoder_get_settings(aenc);
+	for (size_t i = 0; i < MAX_OUTPUT_VIDEO_ENCODERS; i++) {
+		obs_encoder_t *enc =
+			obs_output_get_video_encoder2(stream->output, i);
 
-	const char *codec = obs_encoder_get_codec(venc);
-	stream->video_codec = to_video_type(codec);
+		if (enc) {
+			const char *codec = obs_encoder_get_codec(enc);
+			stream->video_codec[i] = to_video_type(codec);
+		}
+	}
 
 	deque_free(&stream->dbr_frames);
 	stream->audio_bitrate = (long)obs_data_get_int(asettings, "bitrate");
@@ -1371,17 +1421,20 @@ static void *connect_thread(void *data)
 	}
 
 	// HDR streaming disabled for AV1
-	if (stream->video_codec != CODEC_H264 &&
-	    stream->video_codec != CODEC_HEVC) {
-		video_t *video = obs_get_video();
-		const struct video_output_info *info =
-			video_output_get_info(video);
+	for (size_t i = 0; i < MAX_OUTPUT_VIDEO_ENCODERS; i++) {
+		if (stream->video_codec[i] &&
+		    stream->video_codec[i] != CODEC_H264 &&
+		    stream->video_codec[i] != CODEC_HEVC) {
+			video_t *video = obs_get_video();
+			const struct video_output_info *info =
+				video_output_get_info(video);
 
-		if (info->colorspace == VIDEO_CS_2100_HLG ||
-		    info->colorspace == VIDEO_CS_2100_PQ) {
-			obs_output_signal_stop(stream->output,
-					       OBS_OUTPUT_HDR_DISABLED);
-			return NULL;
+			if (info->colorspace == VIDEO_CS_2100_HLG ||
+			    info->colorspace == VIDEO_CS_2100_PQ) {
+				obs_output_signal_stop(stream->output,
+						       OBS_OUTPUT_HDR_DISABLED);
+				return NULL;
+			}
 		}
 	}
 
@@ -1693,7 +1746,7 @@ static void rtmp_stream_data(void *data, struct encoder_packet *packet)
 			stream->got_first_video = true;
 		}
 
-		switch (stream->video_codec) {
+		switch (stream->video_codec[packet->track_idx]) {
 		case CODEC_H264:
 			obs_parse_avc_packet(&new_packet, packet);
 			break;
@@ -1821,7 +1874,7 @@ static int rtmp_stream_connect_time(void *data)
 struct obs_output_info rtmp_output_info = {
 	.id = "rtmp_output",
 	.flags = OBS_OUTPUT_AV | OBS_OUTPUT_ENCODED | OBS_OUTPUT_SERVICE |
-		 OBS_OUTPUT_MULTI_TRACK,
+		 OBS_OUTPUT_MULTI_TRACK_AV,
 #ifdef NO_CRYPTO
 	.protocols = "RTMP",
 #else

plugins/obs-outputs/rtmp-stream.h

@@ -114,7 +114,7 @@ struct rtmp_stream {
 	long dbr_inc_bitrate;
 	bool dbr_enabled;
 
-	enum video_id_t video_codec;
+	enum video_id_t video_codec[MAX_OUTPUT_VIDEO_ENCODERS];
 
 	RTMP rtmp;