rtmp: support additional Enhanced-RTMP features (#3685) (#4088)

new features:
* support publishing Opus and AC-3 tracks
* support publishing more than 2 tracks. This is compatible with OBS multitrack video and OBS VOD audio track

README.md

@@ -26,8 +26,8 @@ Live streams can be published to the server with:
 |[WebRTC servers](#webrtc-servers)|WHEP|AV1, VP9, VP8, [H265](#supported-browsers), H264|Opus, G722, G711 (PCMA, PCMU)|
 |[RTSP clients](#rtsp-clients)|UDP, TCP, RTSPS|AV1, VP9, VP8, H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video, M-JPEG and any RTP-compatible codec|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G726, G722, G711 (PCMA, PCMU), LPCM and any RTP-compatible codec|
 |[RTSP cameras and servers](#rtsp-cameras-and-servers)|UDP, UDP-Multicast, TCP, RTSPS|AV1, VP9, VP8, H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video, M-JPEG and any RTP-compatible codec|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G726, G722, G711 (PCMA, PCMU), LPCM and any RTP-compatible codec|
-|[RTMP clients](#rtmp-clients)|RTMP, RTMPS, Enhanced RTMP|AV1, VP9, H265, H264|MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), G711 (PCMA, PCMU), LPCM|
-|[RTMP cameras and servers](#rtmp-cameras-and-servers)|RTMP, RTMPS, Enhanced RTMP|AV1, VP9, H265, H264|MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), G711 (PCMA, PCMU), LPCM|
+|[RTMP clients](#rtmp-clients)|RTMP, RTMPS, Enhanced RTMP|AV1, VP9, H265, H264|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G711 (PCMA, PCMU), LPCM|
+|[RTMP cameras and servers](#rtmp-cameras-and-servers)|RTMP, RTMPS, Enhanced RTMP|AV1, VP9, H265, H264|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G711 (PCMA, PCMU), LPCM|
 |[HLS cameras and servers](#hls-cameras-and-servers)|Low-Latency HLS, MP4-based HLS, legacy HLS|AV1, VP9, [H265](#supported-browsers-1), H264|Opus, MPEG-4 Audio (AAC)|
 |[UDP/MPEG-TS](#udpmpeg-ts)|Unicast, broadcast, multicast|H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3|
 |[Raspberry Pi Cameras](#raspberry-pi-cameras)||H264||
@@ -2444,9 +2444,10 @@ All the code in this repository is released under the [MIT License](LICENSE). Co
 |----|----|
 |[RTSP / RTP / RTCP specifications](https://github.com/bluenviron/gortsplib#specifications)|RTSP|
 |[HLS specifications](https://github.com/bluenviron/gohlslib#specifications)|HLS|
-|[RTMP](https://rtmp.veriskope.com/pdf/rtmp_specification_1.0.pdf)|RTMP|
-|[Enhanced RTMP v1](https://veovera.org/docs/enhanced/enhanced-rtmp-v1.pdf)|RTMP|
-|[Action Message Format](https://rtmp.veriskope.com/pdf/amf0-file-format-specification.pdf)|RTMP|
+|[Action Message Format - AMF 0](https://veovera.org/docs/legacy/amf0-file-format-spec.pdf)|RTMP|
+|[FLV](https://veovera.org/docs/legacy/video-file-format-v10-1-spec.pdf)|RTMP|
+|[RTMP](https://veovera.org/docs/legacy/rtmp-v1-0-spec.pdf)|RTMP|
+|[Enhanced RTMP v2](https://veovera.org/docs/enhanced/enhanced-rtmp-v2.pdf)|RTMP|
 |[WebRTC: Real-Time Communication in Browsers](https://www.w3.org/TR/webrtc/)|WebRTC|
 |[RFC8835, Transports for WebRTC](https://datatracker.ietf.org/doc/html/rfc8835)|WebRTC|
 |[RFC7742, WebRTC Video Processing and Codec Requirements](https://datatracker.ietf.org/doc/html/rfc7742)|WebRTC|

go.mod

@@ -10,8 +10,8 @@ require (
 	github.com/alecthomas/kong v1.6.0
 	github.com/asticode/go-astits v1.13.0
 	github.com/bluenviron/gohlslib/v2 v2.1.0
-	github.com/bluenviron/gortsplib/v4 v4.12.1-0.20241225145501-66410517c85f
-	github.com/bluenviron/mediacommon v1.13.2
+	github.com/bluenviron/gortsplib/v4 v4.12.1-0.20250101115503-c5485517016a
+	github.com/bluenviron/mediacommon v1.13.3-0.20250101114202-89f364e687ca
 	github.com/datarhei/gosrt v0.8.0
 	github.com/fsnotify/fsnotify v1.8.0
 	github.com/gin-contrib/pprof v1.5.2

go.sum

@@ -33,10 +33,10 @@ github.com/benburkert/openpgp v0.0.0-20160410205803-c2471f86866c h1:8XZeJrs4+ZYh
 github.com/benburkert/openpgp v0.0.0-20160410205803-c2471f86866c/go.mod h1:x1vxHcL/9AVzuk5HOloOEPrtJY0MaalYr78afXZ+pWI=
 github.com/bluenviron/gohlslib/v2 v2.1.0 h1:I0KXXPjnt7QxsR39z97fKe/x1yj22e1NhSqZ5P6FbWE=
 github.com/bluenviron/gohlslib/v2 v2.1.0/go.mod h1:irD+TAdUsb400Gp8v80LKPPC4YumiAieUSO6ICykeWo=
-github.com/bluenviron/gortsplib/v4 v4.12.1-0.20241225145501-66410517c85f h1:x+W67M2KMNCumzQ8XpIFXbVE+quKD/xP5PiaJhKwqTY=
-github.com/bluenviron/gortsplib/v4 v4.12.1-0.20241225145501-66410517c85f/go.mod h1:MwFrCmflxvLTMjgtnPJ2H4SQSB/r9wX8nsR5YPtUs7M=
-github.com/bluenviron/mediacommon v1.13.2 h1:Ssq+59ZtPm5f9iAVVugWNOyl89Vp0G758RMv033lkik=
-github.com/bluenviron/mediacommon v1.13.2/go.mod h1:tffg+sPMErUIe7WMq7ZlYry/rPE6TyENWCrYT5JWcgs=
+github.com/bluenviron/gortsplib/v4 v4.12.1-0.20250101115503-c5485517016a h1:+Pug8A1lj+OEWuJL/54BmMmYNSMl9IyWklwGaeQxn3A=
+github.com/bluenviron/gortsplib/v4 v4.12.1-0.20250101115503-c5485517016a/go.mod h1:MwFrCmflxvLTMjgtnPJ2H4SQSB/r9wX8nsR5YPtUs7M=
+github.com/bluenviron/mediacommon v1.13.3-0.20250101114202-89f364e687ca h1:UIsKAlGy9CH8yW1Kie174+rbOF4V6EAvWZ9EllaObCM=
+github.com/bluenviron/mediacommon v1.13.3-0.20250101114202-89f364e687ca/go.mod h1:RrO01FltoVUlTBGXbOYtmx1ft1oBOpLxfNGsYlaFAO8=
 github.com/bytedance/sonic v1.12.6 h1:/isNmCUF2x3Sh8RAp/4mh4ZGkcFAX/hLrzrK3AvpRzk=
 github.com/bytedance/sonic v1.12.6/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk=
 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=

internal/core/api_test.go

@@ -433,7 +433,10 @@ func TestAPIProtocolListGet(t *testing.T) {
 				conn, err := rtmp.NewClientConn(nconn, u, true)
 				require.NoError(t, err)
 
-				_, err = rtmp.NewWriter(conn, test.FormatH264, nil)
+				w, err := rtmp.NewWriter(conn, test.FormatH264, nil)
+				require.NoError(t, err)
+
+				err = w.WriteH264(2*time.Second, 2*time.Second, [][]byte{{5, 2, 3, 4}})
 				require.NoError(t, err)
 
 				time.Sleep(500 * time.Millisecond)
@@ -1006,7 +1009,10 @@ func TestAPIProtocolKick(t *testing.T) {
 				conn, err := rtmp.NewClientConn(nconn, u, true)
 				require.NoError(t, err)
 
-				_, err = rtmp.NewWriter(conn, test.FormatH264, nil)
+				w, err := rtmp.NewWriter(conn, test.FormatH264, nil)
+				require.NoError(t, err)
+
+				err = w.WriteH264(2*time.Second, 2*time.Second, [][]byte{{5, 2, 3, 4}})
 				require.NoError(t, err)
 
 			case "webrtc":

internal/core/metrics_test.go

@@ -206,8 +206,12 @@ webrtc_sessions_bytes_sent 0
 			conn, err := rtmp.NewClientConn(nconn, u, true)
 			require.NoError(t, err)
 
-			_, err = rtmp.NewWriter(conn, test.FormatH264, nil)
+			w, err := rtmp.NewWriter(conn, test.FormatH264, nil)
 			require.NoError(t, err)
+
+			err = w.WriteH264(2*time.Second, 2*time.Second, [][]byte{{5, 2, 3, 4}})
+			require.NoError(t, err)
+
 			<-terminate
 		}()
 
@@ -223,8 +227,12 @@ webrtc_sessions_bytes_sent 0
 			conn, err := rtmp.NewClientConn(nconn, u, true)
 			require.NoError(t, err)
 
-			_, err = rtmp.NewWriter(conn, test.FormatH264, nil)
+			w, err := rtmp.NewWriter(conn, test.FormatH264, nil)
 			require.NoError(t, err)
+
+			err = w.WriteH264(2*time.Second, 2*time.Second, [][]byte{{5, 2, 3, 4}})
+			require.NoError(t, err)
+
 			<-terminate
 		}()
 
@@ -436,7 +444,7 @@ webrtc_sessions_bytes_sent 0
 		wg.Wait()
 	})
 
-	t.Run("servers deleted", func(t *testing.T) {
+	t.Run("servers disabled", func(t *testing.T) {
 		httpRequest(t, hc, http.MethodPatch, "http://localhost:9997/v3/config/global/patch", map[string]interface{}{
 			"rtsp":   false,
 			"rtmp":   false,

internal/core/path_test.go

@@ -362,6 +362,37 @@ func TestPathRunOnRead(t *testing.T) {
 				require.NoError(t, err)
 				defer source.Close()
 
+				writerDone := make(chan struct{})
+				defer func() { <-writerDone }()
+
+				writerTerminate := make(chan struct{})
+				defer close(writerTerminate)
+
+				go func() {
+					defer close(writerDone)
+					i := 0
+					for {
+						select {
+						case <-time.After(100 * time.Millisecond):
+						case <-writerTerminate:
+							return
+						}
+						err2 := source.WritePacketRTP(media0, &rtp.Packet{
+							Header: rtp.Header{
+								Version:        2,
+								Marker:         true,
+								PayloadType:    96,
+								SequenceNumber: uint16(123 + i),
+								Timestamp:      uint32(45343 + i*90000),
+								SSRC:           563423,
+							},
+							Payload: []byte{5},
+						})
+						require.NoError(t, err2)
+						i++
+					}
+				}()
+
 				switch ca {
 				case "rtsp":
 					reader := gortsplib.Client{}
@@ -426,6 +457,23 @@ func TestPathRunOnRead(t *testing.T) {
 					conn, err := rtmp.NewClientConn(nconn, u, false)
 					require.NoError(t, err)
 
+					go func() {
+						for i := uint16(0); i < 3; i++ {
+							err2 := source.WritePacketRTP(media0, &rtp.Packet{
+								Header: rtp.Header{
+									Version:        2,
+									Marker:         true,
+									PayloadType:    96,
+									SequenceNumber: 123 + i,
+									Timestamp:      45343 + uint32(i)*2*90000,
+									SSRC:           563423,
+								},
+								Payload: []byte{5},
+							})
+							require.NoError(t, err2)
+						}
+					}()
+
 					_, err = rtmp.NewReader(conn)
 					require.NoError(t, err)
 
@@ -455,37 +503,6 @@ func TestPathRunOnRead(t *testing.T) {
 						Log:        test.NilLogger,
 					}
 
-					writerDone := make(chan struct{})
-					defer func() { <-writerDone }()
-
-					writerTerminate := make(chan struct{})
-					defer close(writerTerminate)
-
-					go func() {
-						defer close(writerDone)
-						i := uint16(0)
-						for {
-							select {
-							case <-time.After(100 * time.Millisecond):
-							case <-writerTerminate:
-								return
-							}
-							err2 := source.WritePacketRTP(media0, &rtp.Packet{
-								Header: rtp.Header{
-									Version:        2,
-									Marker:         true,
-									PayloadType:    96,
-									SequenceNumber: 123 + i,
-									Timestamp:      45343,
-									SSRC:           563423,
-								},
-								Payload: []byte{5},
-							})
-							require.NoError(t, err2)
-							i++
-						}
-					}()
-
 					_, err = c.Read(context.Background())
 					require.NoError(t, err)
 					defer checkClose(t, c.Close)

internal/protocols/rtmp/from_stream.go

@@ -86,7 +86,6 @@ func setupVideo(
 				return (*w).WriteH264(
 					timestampToDuration(tunit.PTS, videoFormatH264.ClockRate()),
 					timestampToDuration(dts, videoFormatH264.ClockRate()),
-					idrPresent,
 					tunit.AU)
 			})
 

internal/protocols/rtmp/message/message.go

@@ -18,19 +18,15 @@ const (
 	TypeSetChunkSize     Type = 1
 	TypeAbortMessage     Type = 2
 	TypeAcknowledge      Type = 3
+	TypeUserControl      Type = 4
 	TypeSetWindowAckSize Type = 5
 	TypeSetPeerBandwidth Type = 6
-
-	TypeUserControl Type = 4
-
-	TypeCommandAMF3 Type = 17
-	TypeCommandAMF0 Type = 20
-
-	TypeDataAMF3 Type = 15
-	TypeDataAMF0 Type = 18
-
-	TypeAudio Type = 8
-	TypeVideo Type = 9
+	TypeAudio            Type = 8
+	TypeVideo            Type = 9
+	TypeDataAMF3         Type = 15
+	TypeDataAMF0         Type = 18
+	TypeCommandAMF3      Type = 17
+	TypeCommandAMF0      Type = 20
 )
 
 // UserControlType is a user control type.
@@ -47,27 +43,48 @@ const (
 	UserControlTypePingResponse     UserControlType = 7
 )
 
-// ExtendedType is a message extended type.
-type ExtendedType uint8
+// AudioExType is an audio message extended type.
+type AudioExType uint8
 
-// message extended types.
+// audio message extended types.
 const (
-	ExtendedTypeSequenceStart        ExtendedType = 0
-	ExtendedTypeCodedFrames          ExtendedType = 1
-	ExtendedTypeSequenceEnd          ExtendedType = 2
-	ExtendedTypeFramesX              ExtendedType = 3
-	ExtendedTypeMetadata             ExtendedType = 4
-	ExtendedTypeMPEG2TSSequenceStart ExtendedType = 5
+	AudioExTypeSequenceStart      AudioExType = 0
+	AudioExTypeCodedFrames        AudioExType = 1
+	AudioExTypeSequenceEnd        AudioExType = 2
+	AudioExTypeMultichannelConfig AudioExType = 4
+	AudioExTypeMultitrack         AudioExType = 5
 )
 
-// FourCC is an identifier of a video codec.
+// VideoExType is a video message extended type.
+type VideoExType uint8
+
+// video message extended types.
+const (
+	VideoExTypeSequenceStart        VideoExType = 0
+	VideoExTypeCodedFrames          VideoExType = 1
+	VideoExTypeSequenceEnd          VideoExType = 2
+	VideoExTypeFramesX              VideoExType = 3
+	VideoExTypeMetadata             VideoExType = 4
+	VideoExTypeMPEG2TSSequenceStart VideoExType = 5
+	VideoExTypeMultitrack           VideoExType = 6
+)
+
+// FourCC is an identifier of a Extended-RTMP codec.
 type FourCC uint32
 
-// video codec identifiers.
+// codec identifiers.
 var (
+	// video
 	FourCCAV1  FourCC = 'a'<<24 | 'v'<<16 | '0'<<8 | '1'
 	FourCCVP9  FourCC = 'v'<<24 | 'p'<<16 | '0'<<8 | '9'
 	FourCCHEVC FourCC = 'h'<<24 | 'v'<<16 | 'c'<<8 | '1'
+	FourCCAVC  FourCC = 'a'<<24 | 'v'<<16 | 'c'<<8 | '1'
+
+	// audio
+	FourCCOpus FourCC = 'O'<<24 | 'p'<<16 | 'u'<<8 | 's'
+	FourCCAC3  FourCC = 'a'<<24 | 'c'<<16 | '-'<<8 | '3'
+	FourCCMP4A FourCC = 'm'<<24 | 'p'<<16 | '4'<<8 | 'a'
+	FourCCMP3  FourCC = '.'<<24 | 'm'<<16 | 'p'<<8 | '3'
 )
 
 // Message is a message.