Skip to content

Commit

Permalink
rtmp: support additional Enhanced-RTMP features (#3685)
Browse files Browse the repository at this point in the history
new features:
* Opus and AC-3 streaming
* multitrack streaming, compatible with OBS multitrack video and OBS VOD audio track
  • Loading branch information
aler9 committed Dec 30, 2024
1 parent df3362a commit bba6554
Show file tree
Hide file tree
Showing 44 changed files with 2,423 additions and 972 deletions.
11 changes: 6 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ Live streams can be published to the server with:
|[WebRTC servers](#webrtc-servers)|WHEP|AV1, VP9, VP8, [H265](#supported-browsers), H264|Opus, G722, G711 (PCMA, PCMU)|
|[RTSP clients](#rtsp-clients)|UDP, TCP, RTSPS|AV1, VP9, VP8, H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video, M-JPEG and any RTP-compatible codec|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G726, G722, G711 (PCMA, PCMU), LPCM and any RTP-compatible codec|
|[RTSP cameras and servers](#rtsp-cameras-and-servers)|UDP, UDP-Multicast, TCP, RTSPS|AV1, VP9, VP8, H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video, M-JPEG and any RTP-compatible codec|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G726, G722, G711 (PCMA, PCMU), LPCM and any RTP-compatible codec|
|[RTMP clients](#rtmp-clients)|RTMP, RTMPS, Enhanced RTMP|AV1, VP9, H265, H264|MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), G711 (PCMA, PCMU), LPCM|
|[RTMP cameras and servers](#rtmp-cameras-and-servers)|RTMP, RTMPS, Enhanced RTMP|AV1, VP9, H265, H264|MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), G711 (PCMA, PCMU), LPCM|
|[RTMP clients](#rtmp-clients)|RTMP, RTMPS, Enhanced RTMP|AV1, VP9, H265, H264|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G711 (PCMA, PCMU), LPCM|
|[RTMP cameras and servers](#rtmp-cameras-and-servers)|RTMP, RTMPS, Enhanced RTMP|AV1, VP9, H265, H264|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G711 (PCMA, PCMU), LPCM|
|[HLS cameras and servers](#hls-cameras-and-servers)|Low-Latency HLS, MP4-based HLS, legacy HLS|AV1, VP9, [H265](#supported-browsers-1), H264|Opus, MPEG-4 Audio (AAC)|
|[UDP/MPEG-TS](#udpmpeg-ts)|Unicast, broadcast, multicast|H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3|
|[Raspberry Pi Cameras](#raspberry-pi-cameras)||H264||
Expand Down Expand Up @@ -2438,9 +2438,10 @@ All the code in this repository is released under the [MIT License](LICENSE). Co
|----|----|
|[RTSP / RTP / RTCP specifications](https://github.com/bluenviron/gortsplib#specifications)|RTSP|
|[HLS specifications](https://github.com/bluenviron/gohlslib#specifications)|HLS|
|[RTMP](https://rtmp.veriskope.com/pdf/rtmp_specification_1.0.pdf)|RTMP|
|[Enhanced RTMP v1](https://veovera.org/docs/enhanced/enhanced-rtmp-v1.pdf)|RTMP|
|[Action Message Format](https://rtmp.veriskope.com/pdf/amf0-file-format-specification.pdf)|RTMP|
|[Action Message Format - AMF 0](https://veovera.org/docs/legacy/amf0-file-format-spec.pdf)|RTMP|
|[FLV](https://veovera.org/docs/legacy/video-file-format-v10-1-spec.pdf)|RTMP|
|[RTMP](https://veovera.org/docs/legacy/rtmp-v1-0-spec.pdf)|RTMP|
|[Enhanced RTMP v2](https://veovera.org/docs/enhanced/enhanced-rtmp-v2.pdf)|RTMP|
|[WebRTC: Real-Time Communication in Browsers](https://www.w3.org/TR/webrtc/)|WebRTC|
|[RFC8835, Transports for WebRTC](https://datatracker.ietf.org/doc/html/rfc8835)|WebRTC|
|[RFC7742, WebRTC Video Processing and Codec Requirements](https://datatracker.ietf.org/doc/html/rfc7742)|WebRTC|
Expand Down
10 changes: 8 additions & 2 deletions internal/core/api_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,10 @@ func TestAPIProtocolListGet(t *testing.T) {
conn, err := rtmp.NewClientConn(nconn, u, true)
require.NoError(t, err)

_, err = rtmp.NewWriter(conn, test.FormatH264, nil)
w, err := rtmp.NewWriter(conn, test.FormatH264, nil)
require.NoError(t, err)

err = w.WriteH264(2*time.Second, 2*time.Second, [][]byte{{5, 2, 3, 4}})
require.NoError(t, err)

time.Sleep(500 * time.Millisecond)
Expand Down Expand Up @@ -1006,7 +1009,10 @@ func TestAPIProtocolKick(t *testing.T) {
conn, err := rtmp.NewClientConn(nconn, u, true)
require.NoError(t, err)

_, err = rtmp.NewWriter(conn, test.FormatH264, nil)
w, err := rtmp.NewWriter(conn, test.FormatH264, nil)
require.NoError(t, err)

err = w.WriteH264(2*time.Second, 2*time.Second, [][]byte{{5, 2, 3, 4}})
require.NoError(t, err)

case "webrtc":
Expand Down
14 changes: 11 additions & 3 deletions internal/core/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -206,8 +206,12 @@ webrtc_sessions_bytes_sent 0
conn, err := rtmp.NewClientConn(nconn, u, true)
require.NoError(t, err)

_, err = rtmp.NewWriter(conn, test.FormatH264, nil)
w, err := rtmp.NewWriter(conn, test.FormatH264, nil)
require.NoError(t, err)

err = w.WriteH264(2*time.Second, 2*time.Second, [][]byte{{5, 2, 3, 4}})
require.NoError(t, err)

<-terminate
}()

Expand All @@ -223,8 +227,12 @@ webrtc_sessions_bytes_sent 0
conn, err := rtmp.NewClientConn(nconn, u, true)
require.NoError(t, err)

_, err = rtmp.NewWriter(conn, test.FormatH264, nil)
w, err := rtmp.NewWriter(conn, test.FormatH264, nil)
require.NoError(t, err)

err = w.WriteH264(2*time.Second, 2*time.Second, [][]byte{{5, 2, 3, 4}})
require.NoError(t, err)

<-terminate
}()

Expand Down Expand Up @@ -436,7 +444,7 @@ webrtc_sessions_bytes_sent 0
wg.Wait()
})

t.Run("servers deleted", func(t *testing.T) {
t.Run("servers disabled", func(t *testing.T) {
httpRequest(t, hc, http.MethodPatch, "http://localhost:9997/v3/config/global/patch", map[string]interface{}{
"rtsp": false,
"rtmp": false,
Expand Down
79 changes: 48 additions & 31 deletions internal/core/path_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,37 @@ func TestPathRunOnRead(t *testing.T) {
require.NoError(t, err)
defer source.Close()

writerDone := make(chan struct{})
defer func() { <-writerDone }()

writerTerminate := make(chan struct{})
defer close(writerTerminate)

go func() {
defer close(writerDone)
i := 0
for {
select {
case <-time.After(100 * time.Millisecond):
case <-writerTerminate:
return
}
err2 := source.WritePacketRTP(media0, &rtp.Packet{
Header: rtp.Header{
Version: 2,
Marker: true,
PayloadType: 96,
SequenceNumber: uint16(123 + i),
Timestamp: uint32(45343 + i*90000),
SSRC: 563423,
},
Payload: []byte{5},
})
require.NoError(t, err2)
i++
}
}()

switch ca {
case "rtsp":
reader := gortsplib.Client{}
Expand Down Expand Up @@ -426,6 +457,23 @@ func TestPathRunOnRead(t *testing.T) {
conn, err := rtmp.NewClientConn(nconn, u, false)
require.NoError(t, err)

go func() {
for i := uint16(0); i < 3; i++ {
err2 := source.WritePacketRTP(media0, &rtp.Packet{
Header: rtp.Header{
Version: 2,
Marker: true,
PayloadType: 96,
SequenceNumber: 123 + i,
Timestamp: 45343 + uint32(i)*2*90000,
SSRC: 563423,
},
Payload: []byte{5},
})
require.NoError(t, err2)
}
}()

_, err = rtmp.NewReader(conn)
require.NoError(t, err)

Expand Down Expand Up @@ -455,37 +503,6 @@ func TestPathRunOnRead(t *testing.T) {
Log: test.NilLogger,
}

writerDone := make(chan struct{})
defer func() { <-writerDone }()

writerTerminate := make(chan struct{})
defer close(writerTerminate)

go func() {
defer close(writerDone)
i := uint16(0)
for {
select {
case <-time.After(100 * time.Millisecond):
case <-writerTerminate:
return
}
err2 := source.WritePacketRTP(media0, &rtp.Packet{
Header: rtp.Header{
Version: 2,
Marker: true,
PayloadType: 96,
SequenceNumber: 123 + i,
Timestamp: 45343,
SSRC: 563423,
},
Payload: []byte{5},
})
require.NoError(t, err2)
i++
}
}()

_, err = c.Read(context.Background())
require.NoError(t, err)
defer checkClose(t, c.Close)
Expand Down
1 change: 0 additions & 1 deletion internal/protocols/rtmp/from_stream.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,6 @@ func setupVideo(
return (*w).WriteH264(
timestampToDuration(tunit.PTS, videoFormatH264.ClockRate()),
timestampToDuration(dts, videoFormatH264.ClockRate()),
idrPresent,
tunit.AU)
})

Expand Down
26 changes: 0 additions & 26 deletions internal/protocols/rtmp/message/extended_mpeg2ts_sequence_start.go

This file was deleted.

26 changes: 0 additions & 26 deletions internal/protocols/rtmp/message/extended_sequence_end.go

This file was deleted.

61 changes: 39 additions & 22 deletions internal/protocols/rtmp/message/message.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,15 @@ const (
TypeSetChunkSize Type = 1
TypeAbortMessage Type = 2
TypeAcknowledge Type = 3
TypeUserControl Type = 4
TypeSetWindowAckSize Type = 5
TypeSetPeerBandwidth Type = 6

TypeUserControl Type = 4

TypeCommandAMF3 Type = 17
TypeCommandAMF0 Type = 20

TypeDataAMF3 Type = 15
TypeDataAMF0 Type = 18

TypeAudio Type = 8
TypeVideo Type = 9
TypeAudio Type = 8
TypeVideo Type = 9
TypeDataAMF3 Type = 15
TypeDataAMF0 Type = 18
TypeCommandAMF3 Type = 17
TypeCommandAMF0 Type = 20
)

// UserControlType is a user control type.
Expand All @@ -47,27 +43,48 @@ const (
UserControlTypePingResponse UserControlType = 7
)

// ExtendedType is a message extended type.
type ExtendedType uint8
// AudioExType is an audio message extended type.
type AudioExType uint8

// message extended types.
// audio message extended types.
const (
ExtendedTypeSequenceStart ExtendedType = 0
ExtendedTypeCodedFrames ExtendedType = 1
ExtendedTypeSequenceEnd ExtendedType = 2
ExtendedTypeFramesX ExtendedType = 3
ExtendedTypeMetadata ExtendedType = 4
ExtendedTypeMPEG2TSSequenceStart ExtendedType = 5
AudioExTypeSequenceStart AudioExType = 0
AudioExTypeCodedFrames AudioExType = 1
AudioExTypeSequenceEnd AudioExType = 2
AudioExTypeMultichannelConfig AudioExType = 4
AudioExTypeMultitrack AudioExType = 5
)

// FourCC is an identifier of a video codec.
// VideoExType is a video message extended type.
type VideoExType uint8

// video message extended types.
const (
VideoExTypeSequenceStart VideoExType = 0
VideoExTypeCodedFrames VideoExType = 1
VideoExTypeSequenceEnd VideoExType = 2
VideoExTypeFramesX VideoExType = 3
VideoExTypeMetadata VideoExType = 4
VideoExTypeMPEG2TSSequenceStart VideoExType = 5
VideoExTypeMultitrack VideoExType = 6
)

// FourCC is an identifier of a Extended-RTMP codec.
type FourCC uint32

// video codec identifiers.
// codec identifiers.
var (
// video
FourCCAV1 FourCC = 'a'<<24 | 'v'<<16 | '0'<<8 | '1'
FourCCVP9 FourCC = 'v'<<24 | 'p'<<16 | '0'<<8 | '9'
FourCCHEVC FourCC = 'h'<<24 | 'v'<<16 | 'c'<<8 | '1'
FourCCAVC FourCC = 'a'<<24 | 'v'<<16 | 'c'<<8 | '1'

// audio
FourCCOpus FourCC = 'O'<<24 | 'p'<<16 | 'u'<<8 | 's'
FourCCAC3 FourCC = 'a'<<24 | 'c'<<16 | '-'<<8 | '3'
FourCCMP4A FourCC = 'm'<<24 | 'p'<<16 | '4'<<8 | 'a'
FourCCMP3 FourCC = '.'<<24 | 'm'<<16 | 'p'<<8 | '3'
)

// Message is a message.
Expand Down
File renamed without changes.
61 changes: 61 additions & 0 deletions internal/protocols/rtmp/message/msg_audio_ex_coded_frames.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package message

import (
"fmt"
"time"

"github.com/bluenviron/mediamtx/internal/protocols/rtmp/rawmessage"
)

// AudioExCodedFrames is a CodedFrames extended message.
type AudioExCodedFrames struct {
ChunkStreamID byte
DTS time.Duration
MessageStreamID uint32
FourCC FourCC
Payload []byte
}

func (m *AudioExCodedFrames) unmarshal(raw *rawmessage.Message) error {
if len(raw.Body) < 5 {
return fmt.Errorf("not enough bytes")
}

Check warning on line 22 in internal/protocols/rtmp/message/msg_audio_ex_coded_frames.go

View check run for this annotation

Codecov / codecov/patch

internal/protocols/rtmp/message/msg_audio_ex_coded_frames.go#L21-L22

Added lines #L21 - L22 were not covered by tests

m.ChunkStreamID = raw.ChunkStreamID
m.DTS = raw.Timestamp
m.MessageStreamID = raw.MessageStreamID

m.FourCC = FourCC(raw.Body[1])<<24 | FourCC(raw.Body[2])<<16 | FourCC(raw.Body[3])<<8 | FourCC(raw.Body[4])
switch m.FourCC {
case FourCCOpus, FourCCAC3, FourCCMP4A:
default:
return fmt.Errorf("unsupported fourCC: %v", m.FourCC)

Check warning on line 32 in internal/protocols/rtmp/message/msg_audio_ex_coded_frames.go

View check run for this annotation

Codecov / codecov/patch

internal/protocols/rtmp/message/msg_audio_ex_coded_frames.go#L31-L32

Added lines #L31 - L32 were not covered by tests
}

m.Payload = raw.Body[5:]

return nil
}

func (m AudioExCodedFrames) marshalBodySize() int {
return 5 + len(m.Payload)
}

func (m AudioExCodedFrames) marshal() (*rawmessage.Message, error) {
body := make([]byte, m.marshalBodySize())

body[0] = (9 << 4) | byte(AudioExTypeCodedFrames)
body[1] = uint8(m.FourCC >> 24)
body[2] = uint8(m.FourCC >> 16)
body[3] = uint8(m.FourCC >> 8)
body[4] = uint8(m.FourCC)
copy(body[5:], m.Payload)

return &rawmessage.Message{
ChunkStreamID: m.ChunkStreamID,
Timestamp: m.DTS,
Type: uint8(TypeAudio),
MessageStreamID: m.MessageStreamID,
Body: body,
}, nil
}
Loading

0 comments on commit bba6554

Please sign in to comment.