feat: multiple sourcebuffer

packages/griffith-mp4/src/fmp4/__tests__/fmp4Generator.spec.js

@@ -14,8 +14,12 @@ describe('fmp4Generator', () => {
     ]))
   })
 
-  it('should generator moov buffer size', () => {
-    expect(FMP4.moov(audioData).length).toBe(1154)
+  it('should generator audio moov buffer size', () => {
+    expect(FMP4.moov(audioData, 'audio').length).toBe(652)
+  })
+
+  it('should generator video moov buffer size', () => {
+    expect(FMP4.moov(audioData, 'video').length).toBe(718)
   })
 
   it('should generator moof buffer size', () => {

packages/griffith-mp4/src/fmp4/boxes/__tests__/moov.spec.js

@@ -2,7 +2,11 @@ import moov from '../moov'
 import {audioData} from './__mocks__/data'
 
 describe('moov', () => {
-  it('should get moov box size', () => {
-    expect(moov(audioData).length).toBe(1154)
+  it('should get audio moov box size', () => {
+    expect(moov(audioData, 'audio').length).toBe(652)
+  })
+
+  it('should get video moov box size', () => {
+    expect(moov(audioData, 'video').length).toBe(718)
   })
 })

packages/griffith-mp4/src/fmp4/boxes/moov.js

@@ -3,11 +3,10 @@ import mvhd from './mvhd'
 import trak from './trak'
 import mvex from './mvex'
 
-export default function moov(data) {
+export default function moov(data, type) {
   const content = concatTypedArray(
     mvhd(data),
-    trak({...data, type: 'video'}),
-    trak({...data, type: 'audio'}),
+    trak({...data, type}),
     mvex(data)
   )
   return generateBox('moov', content)

packages/griffith-mp4/src/fmp4/fmp4Generator.js

@@ -11,8 +11,8 @@ export default class FMP4Generator {
     return ftyp()
   }
 
-  static moov(data) {
-    return moov(data)
+  static moov(data, type) {
+    return moov(data, type)
   }
 
   static moof(trackInfo, baseMediaDecodeTime) {

packages/griffith-mp4/src/mp4/boxes/elst.js

@@ -10,7 +10,12 @@ export default function elst(buffer) {
 
   for (let i = 0; i < entryCount; ++i) {
     const segmentDuration = stream.readByte(4)
-    const mediaTime = stream.readByte(4)
+    let mediaTime = stream.readByte(4)
+
+    // 0xffffffff -> -1
+    if (mediaTime === 4294967295) {
+      mediaTime = -1
+    }
     const mediaRateInteger = stream.readByte(2)
     const mediaRateFraction = stream.readByte(2)
 

packages/griffith-mp4/src/mp4/utils/__tests__/findBox.spec.js

@@ -112,4 +112,12 @@ describe('findBox', () => {
   it('find empty', () => {
     expect(findBox(mp4BoxTree)).toEqual({})
   })
+
+  it('find audio elst', () => {
+    expect(findBox(mp4BoxTree, 'audioElst').entries[0].mediaTime).toBe(2048)
+  })
+
+  it('find video elst', () => {
+    expect(findBox(mp4BoxTree, 'videoElst').entries[0].mediaTime).toBe(2048)
+  })
 })

packages/griffith-mp4/src/mp4/utils/__tests__/getSamplesInterval.spec.js

@@ -27,7 +27,7 @@ describe('getSamplesInterval', () => {
       })
     ).toEqual({
       offsetInterVal: [0, 395],
-      timeInterVal: [0, 404480],
+      timeInterVal: [2048, 404480],
     })
   })
 

packages/griffith-mp4/src/mp4/utils/findBox.js

@@ -46,6 +46,10 @@ export default function findBox(mp4BoxTree, type) {
       return findAudioStszBox(mp4BoxTree)
     case 'mp4a':
       return findMp4aBox(mp4BoxTree)
+    case 'audioElst':
+      return findAudioElstBox(mp4BoxTree)
+    case 'videoElst':
+      return findVideoElstBox(mp4BoxTree)
 
     default:
       return {}
@@ -143,3 +147,11 @@ function findAudioStszBox(mp4BoxTree) {
 function findVideoCttsBox(mp4BoxTree) {
   return findVideoStblBox(mp4BoxTree)['ctts']
 }
+
+function findAudioElstBox(mp4BoxTree) {
+  return findAudioTrakBox(mp4BoxTree)['edts']['elst']
+}
+
+function findVideoElstBox(mp4BoxTree) {
+  return findAudioTrakBox(mp4BoxTree)['edts']['elst']
+}

packages/griffith-mp4/src/mp4/utils/getSamplesInterval.js

@@ -44,13 +44,16 @@ export function getAudioSamplesInterval(mp4BoxTree, videoInterval) {
   const {timescale: videoTimescale} = findBox(mp4BoxTree, 'videoMdhd')
   const videoStszBox = findBox(mp4BoxTree, 'videoStsz')
   const audioStszBox = findBox(mp4BoxTree, 'audioStsz')
+  const audioElstBox = findBox(mp4BoxTree, 'audioElst')
 
   const audioStartTime = (startTime / videoTimescale) * audioTimescale
   const audioEndTime = (endTime / videoTimescale) * audioTimescale
 
   let start = 0
   let end = 0
-  let startDuration = 0
+
+  const {mediaTime} = audioElstBox.entries[0]
+  let startDuration = mediaTime || 0
   let endDuration = 0
   for (let i = 0; i < sttsBox.samples.length; i++) {
     const {sampleCount, sampleDelta} = sttsBox.samples[i]

packages/griffith-mp4/src/mse/controller.js

@@ -10,6 +10,14 @@ export default class MSE {
   constructor(video, src) {
     this.video = video
     this.src = src
+    this.sourceBuffers = {
+      video: null,
+      audio: null,
+    }
+    this.mimeTypes = {
+      video: 'video/mp4; codecs="avc1.42E01E"',
+      audio: 'audio/mp4; codecs="mp4a.40.2"',
+    }
     this.installSrc()
   }
 
@@ -20,9 +28,13 @@ export default class MSE {
   }
 
   handleSourceOpen = () => {
-    const mime = 'video/mp4; codecs="avc1.42E01E, mp4a.40.2"'
-    this.sourceBuffer = this.mediaSource.addSourceBuffer(mime)
-    this.sourceBuffer.addEventListener('updateend', () => {
+    this.sourceBuffers.video = this.mediaSource.addSourceBuffer(
+      this.mimeTypes.video
+    )
+    this.sourceBuffers.audio = this.mediaSource.addSourceBuffer(
+      this.mimeTypes.audio
+    )
+    this.sourceBuffers.video.addEventListener('updateend', () => {
       this.mseUpdating = false
 
       if (this.needUpdateTime) {
@@ -73,13 +85,19 @@ export default class MSE {
         this.mp4Probe = new MP4Probe(mp4BoxTreeObject)
         this.mp4BoxTreeObject = mp4BoxTreeObject
 
-        const rawData = concatTypedArray(
+        const videoRawData = concatTypedArray(
+          FMP4.ftyp(),
+          FMP4.moov(this.mp4Probe.mp4Data, 'video')
+        )
+
+        const audioRawData = concatTypedArray(
           FMP4.ftyp(),
-          FMP4.moov(this.mp4Probe.mp4Data)
+          FMP4.moov(this.mp4Probe.mp4Data, 'audio')
         )
 
         this.mediaSource.addEventListener('sourceopen', () => {
-          this.appendBuffer(rawData)
+          this.sourceBuffers.video.appendBuffer(videoRawData)
+          this.sourceBuffers.audio.appendBuffer(audioRawData)
         })
       })
   }
@@ -121,23 +139,24 @@ export default class MSE {
       const {videoInterval, audioInterval} = this.mp4Probe
       const videoBaseMediaDecodeTime = videoInterval.timeInterVal[0]
       const audioBaseMediaDecodeTime = audioInterval.timeInterVal[0]
-      const rawData = concatTypedArray(
+      const videoRawData = concatTypedArray(
         FMP4.moof(videoTrackInfo, videoBaseMediaDecodeTime),
-        FMP4.mdat(videoTrackInfo),
+        FMP4.mdat(videoTrackInfo)
+      )
+
+      const audioRawData = concatTypedArray(
         FMP4.moof(audioTrackInfo, audioBaseMediaDecodeTime),
         FMP4.mdat(audioTrackInfo)
       )
-      this.appendBuffer(rawData)
+
+      this.sourceBuffers.video.appendBuffer(videoRawData)
+      this.sourceBuffers.audio.appendBuffer(audioRawData)
       if (time) {
         this.needUpdateTime = true
       }
     })
   }
 
-  appendBuffer(buffer) {
-    this.sourceBuffer.appendBuffer(buffer)
-  }
-
   loadData(start = 0, end = MAGIC_NUMBER) {
     return new Promise(resolve => {
       new FragmentFetch(this.src, start, end, resolve)
@@ -154,7 +173,6 @@ export default class MSE {
       videoInterval: {offsetInterVal = []} = [],
       mp4Data: {videoSamplesLength},
     } = this.mp4Probe
-
     if (this.mediaSource.readyState === 'open') {
       if (offsetInterVal[1] === videoSamplesLength && !this.mseUpdating) {
         this.destroy()