Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

obs-outputs: Add eRTMP/eFLV multitrack support #10500

Merged
merged 4 commits into from
May 3, 2024
Merged

obs-outputs: Add eRTMP/eFLV multitrack support #10500

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
35665aa
obs-outputs: Always define `CODEC_HEVC`
palana Jan 8, 2024
66d87f9
obs-outputs: Add initial eRTMP multitrack implementation
kc5nra Sep 21, 2023
33bc05e
obs-outputs: Add eRTMP/eFLV support for FLV
kc5nra Sep 29, 2023
b0799b8
obs-outputs: Add `video_id_t` value for 0
palana Nov 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 91 additions & 32 deletions plugins/obs-outputs/flv-mux.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
#define AUDIODATA_AAC 10.0

#define VIDEO_FRAMETYPE_OFFSET 4

enum video_frametype_t {
FT_KEY = 1 << VIDEO_FRAMETYPE_OFFSET,
FT_INTER = 2 << VIDEO_FRAMETYPE_OFFSET,
Expand All @@ -43,10 +44,16 @@ enum packet_type_t {
PACKETTYPE_SEQ_START = 0,
PACKETTYPE_FRAMES = 1,
PACKETTYPE_SEQ_END = 2,
#ifdef ENABLE_HEVC
PACKETTYPE_FRAMESX = 3,
#endif
PACKETTYPE_METADATA = 4
PACKETTYPE_METADATA = 4,
PACKETTYPE_MPEG2TS_SEQ_START = 5,
PACKETTYPE_MULTITRACK = 6
};

enum multitrack_type_t {
MULTITRACKTYPE_ONE_TRACK = 0x00,
MULTITRACKTYPE_MANY_TRACKS = 0x10,
MULTITRACKTYPE_MANY_TRACKS_MANY_CODECS = 0x20,
};

enum datatype_t {
Expand All @@ -59,22 +66,32 @@ enum datatype_t {
static void s_w4cc(struct serializer *s, enum video_id_t id)
{
switch (id) {
case CODEC_NONE:
assert(0 && "Tried to serialize CODEC_NONE");
RytoEX marked this conversation as resolved.
Show resolved Hide resolved
break;

case CODEC_AV1:
s_w8(s, 'a');
s_w8(s, 'v');
s_w8(s, '0');
s_w8(s, '1');
break;
#ifdef ENABLE_HEVC
case CODEC_HEVC:
#ifdef ENABLE_HEVC
s_w8(s, 'h');
s_w8(s, 'v');
s_w8(s, 'c');
s_w8(s, '1');
break;
#else
assert(0);
#endif
case CODEC_H264:
assert(0);
s_w8(s, 'a');
s_w8(s, 'v');
s_w8(s, 'c');
s_w8(s, '1');
break;
}
}

Expand Down Expand Up @@ -361,7 +378,8 @@ void flv_packet_mux(struct encoder_packet *packet, int32_t dts_offset,

// Y2023 spec
void flv_packet_ex(struct encoder_packet *packet, enum video_id_t codec_id,
int32_t dts_offset, uint8_t **output, size_t *size, int type)
int32_t dts_offset, uint8_t **output, size_t *size, int type,
size_t idx)
{
struct array_output_data data;
struct serializer s;
Expand All @@ -371,30 +389,45 @@ void flv_packet_ex(struct encoder_packet *packet, enum video_id_t codec_id,

int32_t time_ms = get_ms_time(packet, packet->dts) - dts_offset;

bool is_multitrack = idx > 0;

// packet head
int header_metadata_size = 5;
#ifdef ENABLE_HEVC
int header_metadata_size = 5; // w8+w4cc
// 3 extra bytes for composition time offset
if (codec_id == CODEC_HEVC && type == PACKETTYPE_FRAMES) {
header_metadata_size = 8;
if ((codec_id == CODEC_H264 || codec_id == CODEC_HEVC) &&
type == PACKETTYPE_FRAMES) {
header_metadata_size += 3; // w24
}
#endif
if (is_multitrack)
header_metadata_size += 2; // w8+w8

s_w8(&s, RTMP_PACKET_TYPE_VIDEO);
s_wb24(&s, (uint32_t)packet->size + header_metadata_size);
s_wtimestamp(&s, time_ms);
s_wb24(&s, 0); // always 0

// packet ext header
s_w8(&s,
FRAME_HEADER_EX | type | (packet->keyframe ? FT_KEY : FT_INTER));
s_w4cc(&s, codec_id);
uint8_t frame_type = packet->keyframe ? FT_KEY : FT_INTER;

#ifdef ENABLE_HEVC
// hevc composition time offset
if (codec_id == CODEC_HEVC && type == PACKETTYPE_FRAMES) {
/*
* We only explicitly emit trackIds iff idx > 0.
* The default trackId is 0.
*/
if (is_multitrack) {
s_w8(&s, FRAME_HEADER_EX | PACKETTYPE_MULTITRACK | frame_type);
s_w8(&s, MULTITRACKTYPE_ONE_TRACK | type);
s_w4cc(&s, codec_id);
// trackId
s_w8(&s, (uint8_t)idx);
} else {
s_w8(&s, FRAME_HEADER_EX | type | frame_type);
s_w4cc(&s, codec_id);
}

// H.264/HEVC composition time offset
if ((codec_id == CODEC_H264 || codec_id == CODEC_HEVC) &&
type == PACKETTYPE_FRAMES) {
s_wb24(&s, get_ms_time(packet, packet->pts - packet->dts));
}
#endif

// packet data
s_write(&s, packet->data, packet->size);
Expand All @@ -407,34 +440,37 @@ void flv_packet_ex(struct encoder_packet *packet, enum video_id_t codec_id,
}

void flv_packet_start(struct encoder_packet *packet, enum video_id_t codec,
uint8_t **output, size_t *size)
uint8_t **output, size_t *size, size_t idx)
{
flv_packet_ex(packet, codec, 0, output, size, PACKETTYPE_SEQ_START);
flv_packet_ex(packet, codec, 0, output, size, PACKETTYPE_SEQ_START,
idx);
}

void flv_packet_frames(struct encoder_packet *packet, enum video_id_t codec,
int32_t dts_offset, uint8_t **output, size_t *size)
int32_t dts_offset, uint8_t **output, size_t *size,
size_t idx)
{
int packet_type = PACKETTYPE_FRAMES;
#ifdef ENABLE_HEVC
// PACKETTYPE_FRAMESX is an optimization to avoid sending composition
// time offsets of 0. See Enhanced RTMP spec.
if (codec == CODEC_HEVC && packet->dts == packet->pts)
if ((codec == CODEC_H264 || codec == CODEC_HEVC) &&
packet->dts == packet->pts)
packet_type = PACKETTYPE_FRAMESX;
#endif
flv_packet_ex(packet, codec, dts_offset, output, size, packet_type);
flv_packet_ex(packet, codec, dts_offset, output, size, packet_type,
idx);
}

void flv_packet_end(struct encoder_packet *packet, enum video_id_t codec,
uint8_t **output, size_t *size)
uint8_t **output, size_t *size, size_t idx)
{
flv_packet_ex(packet, codec, 0, output, size, PACKETTYPE_SEQ_END);
flv_packet_ex(packet, codec, 0, output, size, PACKETTYPE_SEQ_END, idx);
}

void flv_packet_metadata(enum video_id_t codec_id, uint8_t **output,
size_t *size, int bits_per_raw_sample,
uint8_t color_primaries, int color_trc,
int color_space, int min_luminance, int max_luminance)
int color_space, int min_luminance, int max_luminance,
size_t idx)
{
// metadata array
struct array_output_data data;
Expand Down Expand Up @@ -498,16 +534,39 @@ void flv_packet_metadata(enum video_id_t codec_id, uint8_t **output,
s_w8(&s, DATA_TYPE_OBJECT_END);
}

bool is_multitrack = idx > 0;

// packet head
// w8+w4cc
int header_metadata_size = 5;
if (is_multitrack) {
// w8+w8
header_metadata_size += 2;
}

s_w8(&s, RTMP_PACKET_TYPE_VIDEO);
s_wb24(&s, (uint32_t)metadata.bytes.num + 5); // 5 = (w8+w4cc)
s_wb24(&s, (uint32_t)metadata.bytes.num + header_metadata_size);
s_wtimestamp(&s, 0);
s_wb24(&s, 0); // always 0

// packet ext header
// these are the 5 extra bytes mentioned above
s_w8(&s, FRAME_HEADER_EX | PACKETTYPE_METADATA);
s_w4cc(&s, codec_id);
s_w8(&s, FRAME_HEADER_EX | (is_multitrack ? PACKETTYPE_MULTITRACK
: PACKETTYPE_METADATA));

/*
* We only add explicitly emit trackIds iff idx > 0.
* The default trackId is 0.
*/
if (is_multitrack) {
s_w8(&s, MULTITRACKTYPE_ONE_TRACK | PACKETTYPE_METADATA);
s_w4cc(&s, codec_id);
// trackId
s_w8(&s, (uint8_t)idx);
} else {
s_w4cc(&s, codec_id);
}

// packet data
s_write(&s, metadata.bytes.array, metadata.bytes.num);
array_output_serializer_free(&metadata); // must be freed
Expand Down
13 changes: 6 additions & 7 deletions plugins/obs-outputs/flv-mux.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,10 @@
#define MILLISECOND_DEN 1000

enum video_id_t {
CODEC_H264 = 1, // legacy
CODEC_NONE = 0, // not valid in rtmp
CODEC_H264 = 1, // legacy & Y2023 spec
CODEC_AV1, // Y2023 spec
#ifdef ENABLE_HEVC
CODEC_HEVC,
#endif
};

static enum video_id_t to_video_type(const char *codec)
Expand Down Expand Up @@ -62,14 +61,14 @@ extern void flv_additional_packet_mux(struct encoder_packet *packet,
// Y2023 spec
extern void flv_packet_start(struct encoder_packet *packet,
enum video_id_t codec, uint8_t **output,
size_t *size);
size_t *size, size_t idx);
extern void flv_packet_frames(struct encoder_packet *packet,
enum video_id_t codec, int32_t dts_offset,
uint8_t **output, size_t *size);
uint8_t **output, size_t *size, size_t idx);
extern void flv_packet_end(struct encoder_packet *packet, enum video_id_t codec,
uint8_t **output, size_t *size);
uint8_t **output, size_t *size, size_t idx);
extern void flv_packet_metadata(enum video_id_t codec, uint8_t **output,
size_t *size, int bits_per_raw_sample,
uint8_t color_primaries, int color_trc,
int color_space, int min_luminance,
int max_luminance);
int max_luminance, size_t idx);
Loading
Loading