feat(xgplayer-mp4、xgplayer-transmuxer): 支持fmp4 + av1解析播放、seek等能力

packages/xgplayer-mp4-loader/src/loader.js

@@ -3,7 +3,7 @@ import { MP4Parser } from 'xgplayer-transmuxer'
 import { getConfig } from './config'
 import { MediaError } from './error'
 import { Cache } from './cache'
-import { isNumber, moovToMeta, moovToSegments } from './utils'
+import { isNumber, moovToMeta, moovToSegments, sidxToSegments } from './utils'
 import EventEmitter from 'eventemitter3'
 
 export class MP4Loader extends EventEmitter {
@@ -124,15 +124,50 @@ export class MP4Loader extends EventEmitter {
             // throw new MediaError('cannot parse moov box', moov.data)
           }
 
-          const segments = moovToSegments(parsedMoov, this._config.segmentDuration)
+          let segments = moovToSegments(parsedMoov, this._config.segmentDuration)
+
+          // 当box存在但不完整时，补全box
+          const getCompletedBox = async (name) => {
+            const box = MP4Parser.findBox(this.buffer, [name])[0]
+            if (box) {
+              if (box.size > box.data.length) {
+                const res = await this.loadData([box.start, box.start + box.size - 1], cache, config)
+                if (res) {
+                  return MP4Parser.findBox(res.data, [name])[0]
+                }
+              } else {
+                return box
+              }
+            }
+          }
+
+          // 现在的分段式range加载逻辑不适用于fmp4，需要判断太多条件
+          // 因为fmp4的samples信息存放在moof中，而解析moof的range需要依赖sidx
+          // 而sidx和moof的size都是动态的（且sidx不一定存在），导致每个环节都需要判断是否满足解析条件以及对应的兜底处理
+          // todo: 后续加载逻辑需要改为【开区间range+主动取消】才能更好的处理fmp4
+          let isFragmentMP4 = false
+          if (!(segments && segments.videoSegments.length && segments.audioSegments.length)) {
+            const sidx = await getCompletedBox('sidx')
+            if (sidx) {
+              const parsedSidx = MP4Parser.sidx(sidx)
+              if (parsedSidx) {
+                segments = sidxToSegments(parsedMoov, parsedSidx)
+                isFragmentMP4 = true
+              }
+            } else {
+              // 无 sidx box 场景，当前架构只能通过模拟加载完整的fmp4来解析出segments，这样会导致fetch加载的数据量特别大，loading时间长
+              // 更倾向于使用【开区间range+主动取消】方案异步读取，todo
+            }
+          }
+
           if (!segments) {
             this._error = true
             onProgress(null, state, options, {err:'cannot parse segments'})
             return
             // throw new MediaError('cannot parse segments', moov.data)
           }
 
-          this.meta = moovToMeta(parsedMoov)
+          this.meta = moovToMeta(parsedMoov, isFragmentMP4)
           const { videoSegments, audioSegments } = segments
           this.videoSegments = videoSegments
           this.audioSegments = audioSegments
@@ -182,12 +217,26 @@ export class MP4Loader extends EventEmitter {
       throw new MediaError('cannot parse moov box', moov.data)
     }
 
-    const segments = moovToSegments(parsedMoov, this._config.segmentDuration)
+    let segments = moovToSegments(parsedMoov, this._config.segmentDuration)
     if (!segments) {
       throw new MediaError('cannot parse segments', moov.data)
     }
 
-    this.meta = moovToMeta(parsedMoov)
+    let parsedSidx
+    if (!(segments.videoSegments.length && segments.audioSegments.length)) {
+      const moof = MP4Parser.findBox(this.buffer, ['moof'])[0]
+      const sidx = MP4Parser.findBox(this.buffer, ['sidx'])[0]
+      if (moof && moof.size <= moof.data.length && sidx) {
+        const parsedMoof = MP4Parser.moof(moof)
+
+        parsedSidx = MP4Parser.sidx(sidx)
+        if (parsedMoof && parsedSidx) {
+          segments = sidxToSegments(parsedMoov, parsedSidx, parsedMoof)
+        }
+      }
+    }
+
+    this.meta = moovToMeta(parsedMoov, parsedSidx)
     const { videoSegments, audioSegments } = segments
     this.videoSegments = videoSegments
     this.audioSegments = audioSegments

packages/xgplayer-mp4-loader/src/utils.js

@@ -1,3 +1,161 @@
+const TFHDFlag = {
+  BASE_DATA_OFFSET: 1,
+  SAMPLE_DESC: 2,
+  SAMPLE_DUR: 8,
+  SAMPLE_SIZE: 16,
+  SAMPLE_FLAG: 32,
+  DUR_EMPTY: 65536,
+  DEFAULT_BASE_IS_MOOF: 131072
+}
+const TRUNFlag = {
+  DATA_OFFSET: 1,
+  FIRST_FLAG: 4,
+  DURATION: 256,
+  SIZE: 512,
+  FLAG: 1024,
+  CTS_OFFSET: 2048
+}
+const SampleFlag = {
+  DEGRADATION_PRIORITY_MASK: 65535,
+  IS_NON_SYNC: 65536,
+  PADDING_MASK: 917504,
+  REDUNDANCY_MASK: 3145728,
+  DEPENDED_MASK: 12582912,
+  DEPENDS_MASK: 50331648,
+  DEPENDS_NO: 33554432,
+  DEPENDS_YES: 16777216
+}
+
+export function trafToSegments (traf, trex = {}, moofOffset, segDuration, timescale) {
+  const { tfhd, trun, tfdt } = traf
+  const { samples: trunSamples, flags: trunFlags } = trun
+  const { flags: tfhdFlags } = tfhd
+
+  // const defaultSampleDescriptionIndex = tfhdFlags & TFHDFlag.SAMPLE_DESC ? tfhd.sampleDescriptionIndex : (trex.defaultSampleDescriptionIndex || 1)
+  const defaultSampleDuration = tfhdFlags & TFHDFlag.SAMPLE_DUR ? tfhd.defaultSampleDuration : (trex.defaultSampleDuration || 0)
+  const defaultSampleSize = tfhdFlags & TFHDFlag.SAMPLE_SIZE ? tfhd.defaultSampleSize : (trex.defaultSampleSize || 0)
+  const defaultSampleFlags = tfhdFlags & TFHDFlag.SAMPLE_FLAG ? tfhd.defaultSampleFlags : (trex.defaultSampleFlags || 0)
+  const startOffset = tfhdFlags & TFHDFlag.BASE_DATA_OFFSET ? tfhd.baseDataOffset : (tfhdFlags & TFHDFlag.DEFAULT_BASE_IS_MOOF ? moofOffset : 0)
+
+  const frames = []
+  const gops = []
+
+  for (let lastDts = 0, startTime = 0, gopId = 0, totalOffset = startOffset, i = 0; i < trunSamples.length; i++) {
+    const frame = {}
+    frame.index = i
+    frame.size = trunFlags & TRUNFlag.SIZE ? trunSamples[i].size : defaultSampleSize
+    frame.duration = trunFlags & TRUNFlag.DURATION ? trunSamples[i].duration : defaultSampleDuration
+    frame.dts = lastDts > 0 ? lastDts : (tfdt ? tfdt.baseMediaDecodeTime : 0)
+    frame.startTime = startTime
+    if (trunFlags & TRUNFlag.CTS_OFFSET) {
+      frame.pts = frame.dts + trunSamples[i].cts
+    } else {
+      frame.pts = frame.dts
+    }
+    lastDts = frame.dts + frame.duration
+    startTime += frame.duration
+
+    let sampleFlags = defaultSampleFlags
+    if (trunFlags & TRUNFlag.FLAG) {
+      sampleFlags = trunSamples[i].flags
+    } else if (0 === i && trunFlags & TRUNFlag.FIRST_FLAG) {
+      sampleFlags = trun.firstSampleFlag
+    }
+    frame.offset = totalOffset
+    frame.keyframe = !(sampleFlags & (SampleFlag.IS_NON_SYNC | SampleFlag.DEPENDS_YES))
+    totalOffset += frame.size
+    frames.push(frame)
+    if (frame.keyframe) {
+      gopId++
+      gops.push([frame])
+    } else if (gops.length) {
+      gops[gops.length - 1].push(frame)
+    }
+    frame.gopId = gopId
+  }
+
+  const len = frames.length
+  if (!len || (!frames[0].keyframe)) return []
+
+  let time = 0
+  let lastFrame
+  const segments = []
+  const scaledDuration = segDuration * timescale
+  let segmentFrames = []
+
+  // 合并gop至segments，以segDuration作为参考
+  for (let i = 0, len = gops.length; i < len; i++) {
+    time += gops[i].reduce((wret, w) => wret + w.duration, 0)
+    segmentFrames = segmentFrames.concat(gops[i])
+
+    if (time >= scaledDuration || i === gops.length - 1) {
+      lastFrame = segmentFrames[segmentFrames.length - 1]
+      segments.push({
+        index: segments.length,
+        startTime: (segments[segments.length - 1]?.endTime || segmentFrames[0].startTime / timescale),
+        endTime: (lastFrame.startTime + lastFrame.duration) / timescale,
+        duration: time / timescale,
+        range: [segmentFrames[0].offset, lastFrame.offset + lastFrame.size],
+        frames: segmentFrames
+      })
+      time = 0
+      segmentFrames = []
+    }
+  }
+
+  return segments
+}
+
+export function sidxToSegments (moov, sidx) {
+  const tracks = moov.trak
+  if (!tracks || !tracks.length) return
+  const videoTrack = tracks.find(t => t.mdia?.hdlr?.handlerType === 'vide')
+  const audioTrack = tracks.find(t => t.mdia?.hdlr?.handlerType === 'soun')
+  if (!videoTrack && !audioTrack) return
+
+  let audioSegments = []
+  let videoSegments = []
+  if (sidx) {
+    const segments = []
+    let prevTime = 0
+    let prevOffset = sidx.start + sidx.size
+    sidx.references.forEach((ref, i) => {
+      segments.push({
+        index: i,
+        startTime: prevTime,
+        endTime: prevTime + (ref.subsegment_duration / sidx.timescale),
+        duration: ref.subsegment_duration / sidx.timescale,
+        range: [prevOffset, prevOffset + ref.referenced_size],
+        frames: []
+      })
+      prevTime += ref.subsegment_duration / sidx.timescale
+      prevOffset += ref.referenced_size
+    })
+    audioSegments = segments
+    videoSegments = segments
+  } else {
+    // 如果sidx不存在，则代表后续的segments无法通过seek读取
+    // 把整段fmp4当作一个segment，使用开区间range即可
+    const getTrakSegments = (box) => {
+      if (!box) return []
+      return [{
+        index: 0,
+        startTime: 0,
+        endTime: box.duration / box.timescale,
+        duration: box.duration / box.timescale,
+        range: [moov.start + moov.size, ''],
+        frames: []
+      }]
+    }
+    videoSegments = getTrakSegments(moov.mvhd.duration ? moov.mvhd : videoTrack.mdia?.mdhd)
+    audioSegments = getTrakSegments(moov.mvhd.duration ? moov.mvhd : audioTrack.mdia?.mdhd)
+  }
+
+  return {
+    videoSegments,
+    audioSegments
+  }
+}
 
 export function moovToSegments (moov, duration) {
   const tracks = moov.trak
@@ -65,7 +223,7 @@ function getSegments (segDuration, timescale, stts, stsc, stsz, stco, stss, ctts
   let chunkIndex = 0
   let chunkRunIndex = 0
   let offsetInChunk = 0
-  let lastSampleInChunk = stscEntries[0].samplesPerChunk
+  let lastSampleInChunk = stscEntries[0]?.samplesPerChunk
   let lastChunkInRun = stscEntries[1] ? stscEntries[1].firstChunk - 1 : Infinity
   let dts = 0
   let gopId = -1
@@ -118,7 +276,7 @@ function getSegments (segDuration, timescale, stts, stsc, stsz, stco, stss, ctts
   })
 
   const l = frames.length
-  if (!l || (stss && !frames[0].keyframe)) return
+  if (!l || (stss && !frames[0].keyframe)) return []
 
   const segments = []
   let segFrames = []
@@ -166,11 +324,10 @@ function getSegments (segDuration, timescale, stts, stsc, stsz, stco, stss, ctts
       }
     }
   }
-
   return segments
 }
 
-export function moovToMeta (moov) {
+export function moovToMeta (moov, isFragmentMP4) {
   let videoCodec = ''
   let audioCodec = ''
   let width = 0
@@ -197,7 +354,7 @@ export function moovToMeta (moov) {
         width = e1.width
         height = e1.height
         videoTimescale = videoTrack.mdia?.mdhd?.timescale
-        videoCodec = (e1.avcC || e1.hvcC)?.codec
+        videoCodec = (e1.avcC || e1.hvcC || e1.av1C)?.codec
         if (e1.type === 'encv') {
           defaultKID = e1.sinf?.schi?.tenc.default_KID
         }
@@ -226,7 +383,8 @@ export function moovToMeta (moov) {
       audioSampleRate,
       duration,
       audioTimescale,
-      moov
+      moov,
+      isFragmentMP4
     }
   }
 }

packages/xgplayer-mp4/src/mp4.js

@@ -1,6 +1,6 @@
 import EventEmitter from 'eventemitter3'
 import Concat from 'concat-typed-array'
-import { MP4Demuxer, FMP4Remuxer } from 'xgplayer-transmuxer'
+import { MP4Demuxer, FMP4Demuxer, FMP4Remuxer } from 'xgplayer-transmuxer'
 import { ERROR_CODES, NetWorkError, ParserError, ERROR_TYPES } from './error'
 import util from './util'
 import MP4Loader from 'xgplayer-mp4-loader'
@@ -68,6 +68,7 @@ class MP4 extends EventEmitter {
       ...options.reqOptions,
       openLog: checkOpenLog()
     })
+    this.fMP4Demuxer = null
     this.MP4Demuxer = null
     this.FMP4Remuxer = null
     this._needInitSegment = true
@@ -348,6 +349,10 @@ class MP4 extends EventEmitter {
     this.log('>>>>>getSubRange time,',time, JSON.stringify(range))
     if (this.videoTrak) {
       const videoSeg = fragIndex < this.videoTrak.length ? this.videoTrak[fragIndex] : this.videoTrak[this.videoTrak.length - 1]
+      if (videoSeg.frames.length === 0) {
+        this.log('>>>>>getSubRange video, no frames')
+        return range
+      }
       const keyFrameList = videoSeg.frames.filter(getKeyFrameList)
       const videoTimescale = this.meta.videoTimescale
       let startTime = keyFrameList[0].startTime / videoTimescale
@@ -373,6 +378,10 @@ class MP4 extends EventEmitter {
     i = 1
     if (this.audioTrak) {
       const audioSeg = fragIndex < this.audioTrak.length ? this.audioTrak[fragIndex] : this.audioTrak[this.audioTrak.length - 1]
+      if (audioSeg.frames.length === 0) {
+        this.log('>>>>>getSubRange video, no frames')
+        return range
+      }
       const frameList = audioSeg.frames
       const audioTimescale = this.meta.audioTimescale
       i = Math.floor((time * audioTimescale - frameList[0].startTime) / audioSeg.frames[0].duration)
@@ -421,7 +430,7 @@ class MP4 extends EventEmitter {
     const videoIndexRange = this.getSamplesRange(fragIndex, 'video')
     const audioIndexRange = this.getSamplesRange(fragIndex, 'audio')
     const range = [start, start + buffer.byteLength]
-    if (this.transmuxerWorkerControl) {
+    if (this.transmuxerWorkerControl && !this.meta.isFragmentMP4) { // todo: fmp4 demux worker
       const context = {
         range,
         state,
@@ -431,12 +440,20 @@ class MP4 extends EventEmitter {
       this.transmuxerWorkerControl.transmux(this.workerSequence, buffer, start, videoIndexRange, audioIndexRange, this.meta.moov, this.useEME, this.kidValue, context)
     } else {
       try {
-        if (!this.MP4Demuxer) {
-          this.MP4Demuxer = new MP4Demuxer(this.videoTrak, this.audioTrak, null,{openLog: checkOpenLog()})
+        let demuxRet
+        if (this.meta.isFragmentMP4) {
+          if (!this.fMP4Demuxer) {
+            this.fMP4Demuxer = new FMP4Demuxer()
+          }
+          demuxRet = this.fMP4Demuxer.demuxPart(buffer, start, this.meta.moov)
+        } else {
+          if (!this.MP4Demuxer) {
+            this.MP4Demuxer = new MP4Demuxer(this.videoTrak, this.audioTrak, null,{openLog: checkOpenLog()})
+          }
+          demuxRet = this.MP4Demuxer.demuxPart(buffer, start, videoIndexRange, audioIndexRange, this.meta.moov, this.useEME, this.kidValue)
         }
-        const demuxRet = this.MP4Demuxer.demuxPart(buffer, start, videoIndexRange, audioIndexRange, this.meta.moov, this.useEME, this.kidValue)
         if (!this.FMP4Remuxer && (!this.checkCodecH265() || this.options.supportHevc)) {
-          this.FMP4Remuxer = new FMP4Remuxer(this.MP4Demuxer.videoTrack, this.MP4Demuxer.audioTrack, {openLog: checkOpenLog()})
+          this.FMP4Remuxer = new FMP4Remuxer(demuxRet.videoTrack, demuxRet.audioTrack, {openLog: checkOpenLog()})
         }
         let res
         this.log('[mux], videoTimeRange,',demuxRet.videoTrack ? [demuxRet.videoTrack.startPts, demuxRet.videoTrack.endPts] : null, ',audioTimeRange,',demuxRet.audioTrack ? [demuxRet.audioTrack.startPts, demuxRet.audioTrack.endPts] : null)
@@ -481,15 +498,17 @@ class MP4 extends EventEmitter {
     const range = []
     switch (type) {
       case 'video':
-        if (this.videoTrak && fragmentIdx < this.videoTrak.length ) {
+        if (this.videoTrak && fragmentIdx < this.videoTrak.length) {
           const frames = this.videoTrak[fragmentIdx].frames
+          if (!frames.length) break
           range.push(frames[0].index)
           range.push(frames[frames.length - 1].index)
         }
         break
       case 'audio':
         if (this.audioTrak && fragmentIdx < this.audioTrak.length ) {
           const frames = this.audioTrak[fragmentIdx].frames
+          if (!frames.length) break
           range.push(frames[0].index)
           range.push(frames[frames.length - 1].index)
         }