opensource/obs-studio
3
0
Fork 0
mirror of https://github.com/obsproject/obs-studio.git synced 2026-08-29 02:03:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

libobs: Add caption support for AV1

Browse Source 
Extract the itut_t35 caption data buffer from existing caption data
and repackage in a metadata OBU instead of an AVC/HEVC SEI.
This commit is contained in:
John Bowers
2024-03-11 15:06:33 -06:00
committed by Lain
parent 33eb95c274
commit 5c8be1de22
3 changed files with 149 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libobs/obs-av1.c
+89
View File
@@ -63,6 +63,95 @@ static void parse_obu_header(const uint8_t *buf, size_t size, size_t *obu_start,
*obu_start += size_len;
}
// Pass a static 10 byte buffer in. The max size for a leb128.
static inline void encode_uleb128(uint64_t val, uint8_t *out_buf,
size_t *len_out)
{
size_t num_bytes = 0;
uint8_t b = val & 0x7f;
val >>= 7;
while (val > 0) {
out_buf[num_bytes] = b | 0x80;
++num_bytes;
b = val & 0x7f;
val >>= 7;
}
out_buf[num_bytes] = b;
++num_bytes;
*len_out = num_bytes;
}
static const uint8_t METADATA_TYPE_ITUT_T35 = 4;
static const uint8_t OBU_METADATA = 5;
// Create a metadata OBU to carry caption information.
void metadata_obu_itu_t35(const uint8_t *itut_t35_buffer, size_t itut_bufsize,
uint8_t **out_buffer, size_t *outbuf_size)
{
/* From the AV1 spec: 5.3.2 OBU Header Syntax
* -------------
* obu_forbidden_bit (1)
* obu_type (4) // In this case OBS_OBU_METADATA
* obu_extension_flag (1)
* obu_has_size_field (1) // Must be set, size of OBU is variable
* obu_reserved_1bit (1)
* if(obu_extension_flag == 1)
* // skip, because we aren't setting this
*/
uint8_t obu_header_byte = (OBU_METADATA << 3) | (1 << 1);
/* From the AV1 spec: 5.3.1 General OBU Syntax
* if (obu_has_size_field)
* obu_size leb128()
* else
* obu_size = sz - 1 - obu_extension_flag
*
* // Skipping portions unrelated to this OBU type
*
* if (obu_type == OBU_METADATA)
* metdata_obu()
* 5.8.1 General metadata OBU Syntax
* // leb128(metadatatype) should always be 1 byte +1 for trailing bits
* metadata_type leb128()
* 5.8.2 Metadata ITUT T35 syntax
* if (metadata_type == METADATA_TYPE_ITUT_T35)
* // add ITUT T35 payload
* 5.8.1 General metadata OBU Syntax
* // trailing bits will always be 0x80 because
* // everything in here is byte aligned
* trailing_bits( obu_size * 8 - payloadBits )
*/
int64_t size_field = 1 + itut_bufsize + 1;
uint8_t size_buf[10];
size_t size_buf_size = 0;
encode_uleb128(size_field, size_buf, &size_buf_size);
// header + obu_size + metadata_type + metadata_payload + trailing_bits
*outbuf_size = 1 + size_buf_size + 1 + itut_bufsize + 1;
*out_buffer = bzalloc(*outbuf_size);
size_t offset = 0;
(*out_buffer)[0] = obu_header_byte;
++offset;
memcpy((*out_buffer) + offset, size_buf, size_buf_size);
offset += size_buf_size;
(*out_buffer)[offset] = METADATA_TYPE_ITUT_T35;
++offset;
memcpy((*out_buffer) + offset, itut_t35_buffer, itut_bufsize);
offset += itut_bufsize;
/* From AV1 spec: 6.2.1 General OBU semantics
* ... Trailing bits are always present, unless the OBU consists of only
* the header. Trailing bits achieve byte alignment when the payload of
* an OBU is not byte aligned. The trailing bits may also used for
* additional byte padding, and if used are taken into account in the
* sz value. In all cases, the pattern used for the trailing bits
* guarantees that all OBUs (except header-only OBUs) end with the same
* pattern: one bit set to one, optionally followed by zeros. */
(*out_buffer)[offset] = 0x80;
}
bool obs_av1_keyframe(const uint8_t *data, size_t size)
{
const uint8_t *start = data, *end = data + size;
libobs/obs-av1.h
+4
View File
@@ -30,6 +30,10 @@ EXPORT void obs_extract_av1_headers(const uint8_t *packet, size_t size,
size_t *new_packet_size,
uint8_t **header_data, size_t *header_size);
EXPORT void metadata_obu_itu_t35(const uint8_t *itut_t35_buffer,
size_t itut_bufsize, uint8_t **out_buffer,
size_t *outbuf_size);
#ifdef __cplusplus
}
#endif
libobs/obs-output.c
+56 -18
View File
@@ -21,6 +21,7 @@
#include "graphics/math-extra.h"
#include "obs.h"
#include "obs-internal.h"
#include "obs-av1.h"
#include <caption/caption.h>
#include <caption/mpeg.h>
@@ -1510,17 +1511,33 @@ static inline bool has_higher_opposing_ts(struct obs_output *output,
output->highest_audio_ts > packet->dts_usec);
}
static const uint8_t nal_start[4] = {0, 0, 0, 1};
static size_t extract_itut_t35_buffer_from_sei(sei_t *sei, uint8_t **data_out)
{
if (!sei || !sei->head) {
return 0;
}
/* We should only need to get one payload, because the SEI that was
* generated should only have one message, so no need to iterate. If
* we did iterate, we would need to generate multiple OBUs. */
sei_message_t *msg = sei_message_head(sei);
int payload_size = (int)sei_message_size(msg);
uint8_t *payload_data = sei_message_data(msg);
*data_out = malloc(payload_size);
memcpy(*data_out, payload_data, payload_size);
return payload_size;
}
static const uint8_t nal_start[4] = {0, 0, 0, 1};
static bool add_caption(struct obs_output *output, struct encoder_packet *out)
{
struct encoder_packet backup = *out;
sei_t sei;
uint8_t *data;
uint8_t *data = NULL;
size_t size;
long ref = 1;
bool avc = false;
bool hevc = false;
bool av1 = false;
/* Instead of exiting early for unsupported codecs, we will continue
* processing to allow the freeing of caption data even if the captions
@@ -1528,6 +1545,8 @@ static bool add_caption(struct obs_output *output, struct encoder_packet *out)
* the given codec. */
if (strcmp(out->encoder->info.codec, "h264") == 0) {
avc = true;
} else if (strcmp(out->encoder->info.codec, "av1") == 0) {
av1 = true;
#ifdef ENABLE_HEVC
} else if (strcmp(out->encoder->info.codec, "hevc") == 0) {
hevc = true;
@@ -1636,9 +1655,19 @@ static bool add_caption(struct obs_output *output, struct encoder_packet *out)
ctrack->caption_head = next;
}
if (avc || hevc) {
data = malloc(sei_render_size(&sei));
size = sei_render(&sei, data);
if (avc || hevc || av1) {
if (avc || hevc) {
data = malloc(sei_render_size(&sei));
size = sei_render(&sei, data);
}
/* In each of these specs there is an identical structure that
* carries caption information. It is named slightly differently
* in each one. The metadata_itut_t35 in AV1 or the
* user_data_registered_itu_t_t35 in HEVC/AVC. We have an AVC
* SEI wrapped version of that here. We will strip it out and
* repackage it slightly to fit the different codec carrying
* mechanisms. A slightly modified SEI for HEVC and a metadata
* OBU for AV1. */
if (avc) {
/* TODO: SEI should come after AUD/SPS/PPS,
* but before any VCL */
@@ -1646,8 +1675,8 @@ static bool add_caption(struct obs_output *output, struct encoder_packet *out)
da_push_back_array(out_data, data, size);
#ifdef ENABLE_HEVC
} else if (hevc) {
/* Only first NAL, VPS/PPS/SPS should use the 4 byte
start code, SEIs use 3 byte version */
/* Only first NAL (VPS/PPS/SPS) should use the 4 byte
* start code. SEIs use 3 byte version */
da_push_back_array(out_data, nal_start + 1, 3);
/* nal_unit_header( ) {
* forbidden_zero_bit f(1)
@@ -1669,20 +1698,29 @@ static bool add_caption(struct obs_output *output, struct encoder_packet *out)
da_push_back_array(out_data, hevc_nal_header, 2);
da_push_back_array(out_data, &data[1], size - 1);
#endif
} else if (av1) {
uint8_t *obu_buffer = NULL;
size_t obu_buffer_size = 0;
size = extract_itut_t35_buffer_from_sei(&sei, &data);
metadata_obu_itu_t35(data, size, &obu_buffer,
&obu_buffer_size);
if (obu_buffer) {
da_push_back_array(out_data, obu_buffer,
obu_buffer_size);
bfree(obu_buffer);
}
}
free(data);
if (data) {
free(data);
}
obs_encoder_packet_release(out);
*out = backup;
out->data = (uint8_t *)out_data.array + sizeof(ref);
out->size = out_data.num - sizeof(ref);
}
obs_encoder_packet_release(out);
*out = backup;
out->data = (uint8_t *)out_data.array + sizeof(ref);
out->size = out_data.num - sizeof(ref);
sei_free(&sei);
if (avc || hevc) {
return true;
}
return false;
return avc || hevc || av1;
}
static inline void send_interleaved(struct obs_output *output)