Skip to content

Commit

Permalink
muxer: fix invalid fMP4 BaseTime in case of negative DTS
Browse files Browse the repository at this point in the history
  • Loading branch information
@aler9
aler9 committed Nov 19, 2023
1 parent e7092f1 commit c05f34c
Show file tree
Hide file tree
Showing 4 changed files with 217 additions and 106 deletions.
4 changes: 2 additions & 2 deletions muxer_part.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func (p *muxerPart) finalize(nextDTS time.Duration) error {
return nil
}

func (p *muxerPart) writeVideo(sample *augmentedVideoSample) {
func (p *muxerPart) writeVideo(sample *augmentedSample) {
if !p.videoStartDTSFilled {
p.videoStartDTSFilled = true
p.videoStartDTS = sample.dts
Expand All @@ -115,7 +115,7 @@ func (p *muxerPart) writeVideo(sample *augmentedVideoSample) {
p.videoSamples = append(p.videoSamples, &sample.PartSample)
}

func (p *muxerPart) writeAudio(sample *augmentedAudioSample) {
func (p *muxerPart) writeAudio(sample *augmentedSample) {
if !p.audioStartDTSFilled {
p.audioStartDTSFilled = true
p.audioStartDTS = sample.dts
Expand Down
4 changes: 2 additions & 2 deletions muxer_segment_fmp4.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ func (s *muxerSegmentFMP4) finalize(nextDTS time.Duration) error {
}

func (s *muxerSegmentFMP4) writeVideo(
sample *augmentedVideoSample,
sample *augmentedSample,
nextDTS time.Duration,
adjustedPartDuration time.Duration,
) error {
Expand Down Expand Up @@ -168,7 +168,7 @@ func (s *muxerSegmentFMP4) writeVideo(
}

func (s *muxerSegmentFMP4) writeAudio(
sample *augmentedAudioSample,
sample *augmentedSample,
nextAudioSampleDTS time.Duration,
adjustedPartDuration time.Duration,
) error {
Expand Down
167 changes: 92 additions & 75 deletions muxer_segmenter_fmp4.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ import (
"github.com/bluenviron/gohlslib/pkg/storage"
)

const (
fmp4StartDTS = 10 * time.Second
)

func fmp4TimeScale(c codecs.Codec) uint32 {
switch codec := c.(type) {
case *codecs.MPEG4Audio:
Expand Down Expand Up @@ -77,13 +81,7 @@ func allocateDTSExtractor(track *Track) dtsExtractor {
return nil
}

type augmentedVideoSample struct {
fmp4.PartSample
dts time.Duration
ntp time.Time
}

type augmentedAudioSample struct {
type augmentedSample struct {
fmp4.PartSample
dts time.Duration
ntp time.Time
Expand All @@ -104,11 +102,12 @@ type muxerSegmenterFMP4 struct {
audioTimeScale uint32
videoFirstRandomAccessReceived bool
videoDTSExtractor dtsExtractor
startDTS time.Duration
currentSegment *muxerSegmentFMP4
nextSegmentID uint64
nextPartID uint64
nextVideoSample *augmentedVideoSample
nextAudioSample *augmentedAudioSample
nextVideoSample *augmentedSample
nextAudioSample *augmentedSample
firstSegmentFinalized bool
sampleDurations map[time.Duration]struct{}
adjustedPartDuration time.Duration
Expand Down Expand Up @@ -219,16 +218,14 @@ func (m *muxerSegmenterFMP4) writeAV1(
return err
}

sample := &augmentedVideoSample{
PartSample: *ps,
dts: dts,
ntp: ntp,
}

return m.writeVideo(
randomAccess,
forceSwitch,
sample)
&augmentedSample{
PartSample: *ps,
dts: dts,
ntp: ntp,
})

Check warning on line 228 in muxer_segmenter_fmp4.go

View check run for this annotation

Codecov / codecov/patch

muxer_segmenter_fmp4.go#L224-L228 

Added lines #L224 - L228 were not covered by tests
}

func (m *muxerSegmenterFMP4) writeVP9(
Expand All @@ -247,19 +244,17 @@ func (m *muxerSegmenterFMP4) writeVP9(
m.videoFirstRandomAccessReceived = true
}

sample := &augmentedVideoSample{
PartSample: fmp4.PartSample{
IsNonSyncSample: !randomAccess,
Payload: frame,
},
dts: dts,
ntp: ntp,
}

return m.writeVideo(
randomAccess,
forceSwitch,
sample)
&augmentedSample{
PartSample: fmp4.PartSample{
IsNonSyncSample: !randomAccess,
Payload: frame,
},
dts: dts,
ntp: ntp,
})

Check warning on line 257 in muxer_segmenter_fmp4.go

View check run for this annotation

Codecov / codecov/patch

muxer_segmenter_fmp4.go#L250-L257 

Added lines #L250 - L257 were not covered by tests
}

func (m *muxerSegmenterFMP4) writeH26x(
Expand Down Expand Up @@ -295,23 +290,74 @@ func (m *muxerSegmenterFMP4) writeH26x(
return err
}

sample := &augmentedVideoSample{
PartSample: *ps,
dts: dts,
ntp: ntp,
}

return m.writeVideo(
randomAccess,
forceSwitch,
sample)
&augmentedSample{
PartSample: *ps,
dts: dts,
ntp: ntp,
})
}

func (m *muxerSegmenterFMP4) writeOpus(ntp time.Time, pts time.Duration, packets [][]byte) error {
for _, packet := range packets {
err := m.writeAudio(&augmentedSample{
PartSample: fmp4.PartSample{
Payload: packet,
},
dts: pts,
ntp: ntp,
})
if err != nil {
return err
}

Check warning on line 314 in muxer_segmenter_fmp4.go

View check run for this annotation

Codecov / codecov/patch

muxer_segmenter_fmp4.go#L303-L314 

Added lines #L303 - L314 were not covered by tests

duration := opus.PacketDuration(packet)
ntp = ntp.Add(duration)
pts += duration

Check warning on line 318 in muxer_segmenter_fmp4.go

View check run for this annotation

Codecov / codecov/patch

muxer_segmenter_fmp4.go#L316-L318 

Added lines #L316 - L318 were not covered by tests
}

return nil

Check warning on line 321 in muxer_segmenter_fmp4.go

View check run for this annotation

Codecov / codecov/patch

muxer_segmenter_fmp4.go#L321 

Added line #L321 was not covered by tests
}

func (m *muxerSegmenterFMP4) writeMPEG4Audio(ntp time.Time, pts time.Duration, aus [][]byte) error {
sampleRate := time.Duration(m.audioTrack.Codec.(*codecs.MPEG4Audio).Config.SampleRate)

for i, au := range aus {
auNTP := ntp.Add(time.Duration(i) * mpeg4audio.SamplesPerAccessUnit *
time.Second / sampleRate)
auPTS := pts + time.Duration(i)*mpeg4audio.SamplesPerAccessUnit*
time.Second/sampleRate

err := m.writeAudio(&augmentedSample{
PartSample: fmp4.PartSample{
Payload: au,
},
dts: auPTS,
ntp: auNTP,
})
if err != nil {
return err
}

Check warning on line 342 in muxer_segmenter_fmp4.go

View check run for this annotation

Codecov / codecov/patch

muxer_segmenter_fmp4.go#L341-L342 

Added lines #L341 - L342 were not covered by tests
}

return nil
}

func (m *muxerSegmenterFMP4) writeVideo(
randomAccess bool,
forceSwitch bool,
sample *augmentedVideoSample,
sample *augmentedSample,
) error {
// add a starting DTS to avoid a negative BaseTime
sample.dts += fmp4StartDTS

// BaseTime is still negative, this is not supported by fMP4. Reject the sample silently.
if (sample.dts - m.startDTS) < 0 {
return nil
}

Check warning on line 359 in muxer_segmenter_fmp4.go

View check run for this annotation

Codecov / codecov/patch

muxer_segmenter_fmp4.go#L358-L359 

Added lines #L358 - L359 were not covered by tests

// put samples into a queue in order to
// - compute sample duration
// - check if next sample is IDR
Expand All @@ -321,8 +367,10 @@ func (m *muxerSegmenterFMP4) writeVideo(
}
sample.Duration = uint32(durationGoToMp4(m.nextVideoSample.dts-sample.dts, 90000))

// create first segment
if m.currentSegment == nil {
// create first segment
m.startDTS = sample.dts

var err error
m.currentSegment, err = newMuxerSegmentFMP4(
m.lowLatency,
Expand Down Expand Up @@ -399,55 +447,22 @@ func (m *muxerSegmenterFMP4) writeVideo(
return nil
}

func (m *muxerSegmenterFMP4) writeOpus(ntp time.Time, pts time.Duration, packets [][]byte) error {
for _, packet := range packets {
err := m.writeAudio(ntp, pts, packet)
if err != nil {
return err
}

duration := opus.PacketDuration(packet)
ntp = ntp.Add(duration)
pts += duration
}

return nil
}
func (m *muxerSegmenterFMP4) writeAudio(sample *augmentedSample) error {
// add a starting DTS to avoid a negative BaseTime
sample.dts += fmp4StartDTS

func (m *muxerSegmenterFMP4) writeMPEG4Audio(ntp time.Time, pts time.Duration, aus [][]byte) error {
sampleRate := time.Duration(m.audioTrack.Codec.(*codecs.MPEG4Audio).Config.SampleRate)

for i, au := range aus {
auNTP := ntp.Add(time.Duration(i) * mpeg4audio.SamplesPerAccessUnit *
time.Second / sampleRate)
auPTS := pts + time.Duration(i)*mpeg4audio.SamplesPerAccessUnit*
time.Second/sampleRate

err := m.writeAudio(auNTP, auPTS, au)
if err != nil {
return err
}
// BaseTime is still negative, this is not supported by fMP4. Reject the sample silently.
if (sample.dts - m.startDTS) < 0 {
return nil
}

return nil
}

func (m *muxerSegmenterFMP4) writeAudio(ntp time.Time, dts time.Duration, au []byte) error {
if m.videoTrack != nil {
// wait for the video track
if !m.videoFirstRandomAccessReceived {
return nil
}
}

sample := &augmentedAudioSample{
PartSample: fmp4.PartSample{
Payload: au,
},
dts: dts,
ntp: ntp,
}

// put samples into a queue in order to compute the sample duration
sample, m.nextAudioSample = m.nextAudioSample, sample
if sample == nil {
Expand All @@ -456,8 +471,10 @@ func (m *muxerSegmenterFMP4) writeAudio(ntp time.Time, dts time.Duration, au []b
sample.Duration = uint32(durationGoToMp4(m.nextAudioSample.dts-sample.dts, m.audioTimeScale))

if m.videoTrack == nil {
// create first segment
if m.currentSegment == nil {
// create first segment
m.startDTS = sample.dts

var err error
m.currentSegment, err = newMuxerSegmentFMP4(
m.lowLatency,
Expand Down
Loading

0 comments on commit c05f34c

Please sign in to comment.