mac-vth264: Manually mark priority bits for frames

This commit is the mac-vth264 version of edfc2be in obs-qsv11.

plugins/mac-vth264/encoder.c

@@ -1,5 +1,6 @@
 #include <obs-module.h>
 #include <util/darray.h>
+#include <obs-avc.h>
 
 #include <CoreFoundation/CoreFoundation.h>
 #include <VideoToolbox/VideoToolbox.h>
@@ -671,6 +672,10 @@ static bool is_sample_keyframe(CMSampleBufferRef buffer)
 static bool parse_sample(struct vt_h264_encoder *enc, CMSampleBufferRef buffer,
 			 struct encoder_packet *packet, CMTime off)
 {
+	uint8_t *start;
+	uint8_t *end;
+	int type;
+
 	CMTime pts = CMSampleBufferGetPresentationTimeStamp(buffer);
 	CMTime dts = CMSampleBufferGetDecodeTimeStamp(buffer);
 
@@ -703,6 +708,37 @@ static bool parse_sample(struct vt_h264_encoder *enc, CMSampleBufferRef buffer,
 	packet->size = enc->packet_data.num;
 	packet->keyframe = keyframe;
 
+	/* ------------------------------------ */
+
+	start = enc->packet_data.array;
+	end = start + enc->packet_data.num;
+
+	start = (uint8_t *)obs_avc_find_startcode(start, end);
+	while (true) {
+		while (start < end && !*(start++))
+			;
+
+		if (start == end)
+			break;
+
+		type = start[0] & 0x1F;
+		if (type == OBS_NAL_SLICE_IDR || type == OBS_NAL_SLICE) {
+			uint8_t prev_type = (start[0] >> 5) & 0x3;
+			start[0] &= ~(3 << 5);
+
+			if (type & OBS_NAL_SLICE)
+				start[0] |= OBS_NAL_PRIORITY_HIGHEST << 5;
+			else if (type & OBS_NAL_SLICE_IDR)
+				start[0] |= OBS_NAL_PRIORITY_HIGH << 5;
+			else
+				start[0] |= prev_type << 5;
+		}
+
+		start = (uint8_t *)obs_avc_find_startcode(start, end);
+	}
+
+	/* ------------------------------------ */
+
 	CFRelease(buffer);
 	return true;