feat(HLS): Pick up an audio track from the original video

externs/shaka/manifest.js

@@ -387,7 +387,8 @@ shaka.extern.SegmentIndex = class {
  *   mssPrivateData: (shaka.extern.MssPrivateData|undefined),
  *   external: boolean,
  *   fastSwitching: boolean,
- *   fullMimeTypes: !Set.<string>
+ *   fullMimeTypes: !Set.<string>,
+ *   isAudioMuxedInVideo: boolean
  * }}
  *
  * @description
@@ -534,6 +535,8 @@ shaka.extern.SegmentIndex = class {
  *   represents the types used in each period of the original manifest.
  *   Meant for being used by compatibility checking, such as with
  *   MediaSource.isTypeSupported.
+ * @property {boolean} isAudioMuxedInVideo
+ *   Indicate if the audio of this stream is muxed in the video of other stream.
  *
  * @exportDoc
  */

externs/shaka/offline.js

@@ -142,7 +142,8 @@ shaka.extern.ManifestDB;
  *   closedCaptions: Map.<string, string>,
  *   tilesLayout: (string|undefined),
  *   external: boolean,
- *   fastSwitching: boolean
+ *   fastSwitching: boolean,
+ *   isAudioMuxedInVideo: boolean
  * }}
  *
  * @property {number} id
@@ -218,6 +219,8 @@ shaka.extern.ManifestDB;
  *   Eg: external text tracks.
  * @property {boolean} fastSwitching
  *   Indicate if the stream should be used for fast switching.
+ * @property {boolean} isAudioMuxedInVideo
+ *   Indicate if the audio of this stream is muxed in the video of other stream.
  */
 shaka.extern.StreamDB;
 

lib/dash/dash_parser.js

@@ -2384,6 +2384,7 @@ shaka.dash.DashParser = class {
         fastSwitching: false,
         fullMimeTypes: new Set([shaka.util.MimeUtils.getFullType(
             context.representation.mimeType, context.representation.codecs)]),
+        isAudioMuxedInVideo: false,
       };
     }
 

lib/hls/hls_parser.js

@@ -448,6 +448,9 @@ shaka.hls.HlsParser = class {
    * @private
    */
   async updateStream_(streamInfo) {
+    if (streamInfo.stream.isAudioMuxedInVideo) {
+      return;
+    }
     const manifestUris = [];
     for (const uri of streamInfo.getUris()) {
       const uriObj = new goog.Uri(uri);
@@ -1491,11 +1494,11 @@ shaka.hls.HlsParser = class {
    * @private
    */
   createStreamInfosFromMediaTags_(mediaTags, groupIdPathwayIdMapping) {
-    // Filter out subtitles and  media tags without uri.
+    // Filter out subtitles and media tags without uri (except audio).
     mediaTags = mediaTags.filter((tag) => {
       const uri = tag.getAttributeValue('URI') || '';
       const type = tag.getAttributeValue('TYPE');
-      return type != 'SUBTITLES' && uri != '';
+      return type != 'SUBTITLES' && (uri != '' || type == 'AUDIO');
     });
 
     const groupedTags = {};
@@ -1511,7 +1514,7 @@ shaka.hls.HlsParser = class {
     for (const key in groupedTags) {
       // Create stream info for each audio / video media grouped tag.
       this.createStreamInfoFromMediaTags_(
-          groupedTags[key], groupIdPathwayIdMapping);
+          groupedTags[key], groupIdPathwayIdMapping, /* requireUri= */ false);
     }
   }
 
@@ -1781,32 +1784,8 @@ shaka.hls.HlsParser = class {
     }
 
     if (!ignoreStream) {
-      let language = null;
-      let name = null;
-      let channelsCount = null;
-      let spatialAudio = false;
-      let characteristics = null;
-      let sampleRate = null;
-      if (!streamInfos.length) {
-        const mediaTag = mediaTags.find((tag) => {
-          const uri = tag.getAttributeValue('URI') || '';
-          const type = tag.getAttributeValue('TYPE');
-          const groupId = tag.getRequiredAttrValue('GROUP-ID');
-          return type != 'SUBTITLES' && uri == '' &&
-              globalGroupIds.includes(groupId);
-        });
-        if (mediaTag) {
-          language = mediaTag.getAttributeValue('LANGUAGE');
-          name = mediaTag.getAttributeValue('NAME');
-          channelsCount = this.getChannelsCount_(mediaTag);
-          spatialAudio = this.isSpatialAudio_(mediaTag);
-          characteristics = mediaTag.getAttributeValue('CHARACTERISTICS');
-          sampleRate = this.getSampleRate_(mediaTag);
-        }
-      }
-      const streamInfo = this.createStreamInfoFromVariantTags_(
-          tags, allCodecs, type, language, name, channelsCount,
-          characteristics, sampleRate, spatialAudio);
+      const streamInfo =
+          this.createStreamInfoFromVariantTags_(tags, allCodecs, type);
       if (globalGroupId) {
         streamInfo.stream.groupId = globalGroupId;
       }
@@ -2142,17 +2121,20 @@ shaka.hls.HlsParser = class {
    *
    * @param {!Array.<!shaka.hls.Tag>} tags
    * @param {!Map.<string, string>} groupIdPathwayIdMapping
+   * @param {boolean=} requireUri
    * @return {!shaka.hls.HlsParser.StreamInfo}
    * @private
    */
-  createStreamInfoFromMediaTags_(tags, groupIdPathwayIdMapping) {
+  createStreamInfoFromMediaTags_(tags, groupIdPathwayIdMapping,
+      requireUri = true) {
     const verbatimMediaPlaylistUris = [];
     const globalGroupIds = [];
     const groupIdUriMappping = new Map();
     for (const tag of tags) {
       goog.asserts.assert(tag.name == 'EXT-X-MEDIA',
           'Should only be called on media tags!');
-      const uri = tag.getRequiredAttrValue('URI');
+      const uri = requireUri ? tag.getRequiredAttrValue('URI') :
+          (tag.getAttributeValue('URI') || shaka.hls.HlsParser.FAKE_MUXED_URL_);
       const groupId = tag.getRequiredAttrValue('GROUP-ID');
       verbatimMediaPlaylistUris.push(uri);
       globalGroupIds.push(groupId);
@@ -2352,17 +2334,10 @@ shaka.hls.HlsParser = class {
    * @param {!Array.<!shaka.hls.Tag>} tags
    * @param {!Array.<string>} allCodecs
    * @param {string} type
-   * @param {?string} language
-   * @param {?string} name
-   * @param {?number} channelsCount
-   * @param {?string} characteristics
-   * @param {?number} sampleRate
-   * @param {boolean} spatialAudio
    * @return {!shaka.hls.HlsParser.StreamInfo}
    * @private
    */
-  createStreamInfoFromVariantTags_(tags, allCodecs, type, language, name,
-      channelsCount, characteristics, sampleRate, spatialAudio) {
+  createStreamInfoFromVariantTags_(tags, allCodecs, type) {
     const streamId = this.globalId_++;
     const verbatimMediaPlaylistUris = [];
     for (const tag of tags) {
@@ -2384,10 +2359,11 @@ shaka.hls.HlsParser = class {
     const closedCaptions = this.getClosedCaptions_(tags[0], type);
     const codecs = shaka.util.ManifestParserUtils.guessCodecs(type, allCodecs);
     const streamInfo = this.createStreamInfo_(
-        streamId, verbatimMediaPlaylistUris, codecs, type, language,
-        /* primary= */ false, name, channelsCount, closedCaptions,
-        characteristics, /* forced= */ false, sampleRate,
-        /* spatialAudio= */ false);
+        streamId, verbatimMediaPlaylistUris,
+        codecs, type, /* language= */ null, /* primary= */ false,
+        /* name= */ null, /* channelcount= */ null, closedCaptions,
+        /* characteristics= */ null, /* forced= */ false,
+        /* sampleRate= */ null, /* spatialAudio= */ false);
 
     this.uriToStreamInfosMap_.set(key, streamInfo);
     return streamInfo;
@@ -2423,6 +2399,15 @@ shaka.hls.HlsParser = class {
         languageValue, primary, name, channelsCount, closedCaptions,
         characteristics, forced, sampleRate, spatialAudio);
 
+    const FAKE_MUXED_URL_ = shaka.hls.HlsParser.FAKE_MUXED_URL_;
+    if (verbatimMediaPlaylistUris.includes(FAKE_MUXED_URL_)) {
+      stream.isAudioMuxedInVideo = true;
+      // We assigned the TS mimetype because it is the only one that works
+      // with this functionality. MP4 is not supported right now.
+      stream.mimeType = 'video/mp2t';
+      this.setFullTypeForStream_(stream);
+    }
+
     const redirectUris = [];
     const getUris = () => {
       if (this.contentSteeringManager_ &&
@@ -2666,6 +2651,12 @@ shaka.hls.HlsParser = class {
         return creationPromise;
       }
 
+      if (stream.isAudioMuxedInVideo) {
+        const segmentIndex = new shaka.media.SegmentIndex([]);
+        stream.segmentIndex = segmentIndex;
+        return Promise.resolve();
+      }
+
       // Create a new PendingRequest to be able to cancel this specific
       // download.
       pendingRequest = this.requestManifest_(streamInfo.getUris(),
@@ -2706,7 +2697,8 @@ shaka.hls.HlsParser = class {
   getMinDuration_() {
     let minDuration = Infinity;
     for (const streamInfo of this.uriToStreamInfosMap_.values()) {
-      if (streamInfo.stream.segmentIndex && streamInfo.stream.type != 'text') {
+      if (streamInfo.stream.segmentIndex && streamInfo.stream.type != 'text' &&
+          !streamInfo.stream.isAudioMuxedInVideo) {
         // Since everything is already offset to 0 (either by sync or by being
         // VOD), only maxTimestamp is necessary to compute the duration.
         minDuration = Math.min(minDuration, streamInfo.maxTimestamp);
@@ -2723,7 +2715,8 @@ shaka.hls.HlsParser = class {
     let maxTimestamp = Infinity;
     let minTimestamp = Infinity;
     for (const streamInfo of this.uriToStreamInfosMap_.values()) {
-      if (streamInfo.stream.segmentIndex && streamInfo.stream.type != 'text') {
+      if (streamInfo.stream.segmentIndex && streamInfo.stream.type != 'text' &&
+          !streamInfo.stream.isAudioMuxedInVideo) {
         maxTimestamp = Math.min(maxTimestamp, streamInfo.maxTimestamp);
         minTimestamp = Math.min(minTimestamp, streamInfo.minTimestamp);
       }
@@ -3119,6 +3112,7 @@ shaka.hls.HlsParser = class {
       external: false,
       fastSwitching: false,
       fullMimeTypes: new Set(),
+      isAudioMuxedInVideo: false,
     };
     this.setFullTypeForStream_(stream);
     return stream;
@@ -5148,6 +5142,14 @@ shaka.hls.HlsParser.PresentationType_ = {
   LIVE: 'LIVE',
 };
 
+
+/**
+ * @const {string}
+ * @private
+ */
+shaka.hls.HlsParser.FAKE_MUXED_URL_ = 'shaka://hls-muxed';
+
+
 shaka.media.ManifestParser.registerParserByMime(
     'application/x-mpegurl', () => new shaka.hls.HlsParser());
 shaka.media.ManifestParser.registerParserByMime(

lib/media/media_source_engine.js

@@ -510,6 +510,10 @@ shaka.media.MediaSourceEngine = class {
         this.queues_[contentType] = [];
       }
     }
+    const audio = streamsByType.get(ContentType.AUDIO);
+    if (audio && audio.isAudioMuxedInVideo) {
+      this.needSplitMuxedContent_ = true;
+    }
   }
 
   /**
@@ -1951,6 +1955,10 @@ shaka.media.MediaSourceEngine = class {
           this.queues_[contentType] = [];
         }
       }
+      const audio = streamsByType.get(ContentType.AUDIO);
+      if (audio && audio.isAudioMuxedInVideo) {
+        this.needSplitMuxedContent_ = true;
+      }
 
       // Fake a seek to catchup the playhead.
       this.video_.currentTime = currentTime;

lib/media/streaming_engine.js

@@ -622,13 +622,21 @@ shaka.media.StreamingEngine = class {
       }
     }
 
+    const shouldResetMediaSource =
+        mediaState.stream.isAudioMuxedInVideo != stream.isAudioMuxedInVideo;
+
     mediaState.stream = stream;
     mediaState.segmentIterator = null;
     mediaState.adaptation = !!adaptation;
 
     const streamTag = shaka.media.StreamingEngine.logPrefix_(mediaState);
     shaka.log.debug('switch: switching to Stream ' + streamTag);
 
+    if (shouldResetMediaSource) {
+      this.resetMediaSource(/* force= */ true, /* clearBuffer= */ false);
+      return;
+    }
+
     if (clearBuffer) {
       if (mediaState.clearingBuffer) {
         // We are already going to clear the buffer, but make sure it is also
@@ -1313,6 +1321,10 @@ shaka.media.StreamingEngine = class {
       this.playerInterface_.mediaSourceEngine.clearSelectedClosedCaptionId();
     }
 
+    if (mediaState.stream.isAudioMuxedInVideo) {
+      return null;
+    }
+
     if (!this.playerInterface_.mediaSourceEngine.isStreamingAllowed() &&
         mediaState.type != ContentType.TEXT) {
       // It is not allowed to add segments yet, so we schedule an update to
@@ -2866,27 +2878,34 @@ shaka.media.StreamingEngine = class {
   /**
    * Reset Media Source
    *
+   * @param {boolean=} force
    * @return {!Promise.<boolean>}
    */
-  async resetMediaSource() {
+  async resetMediaSource(force = false, clearBuffer = true) {
     const now = (Date.now() / 1000);
     const minTimeBetweenRecoveries = this.config_.minTimeBetweenRecoveries;
-    if (!this.config_.allowMediaSourceRecoveries ||
-        (now - this.lastMediaSourceReset_) < minTimeBetweenRecoveries) {
-      return false;
+    if (!force) {
+      if (!this.config_.allowMediaSourceRecoveries ||
+          (now - this.lastMediaSourceReset_) < minTimeBetweenRecoveries) {
+        return false;
+      }
+      this.lastMediaSourceReset_ = now;
     }
-    this.lastMediaSourceReset_ = now;
     const ContentType = shaka.util.ManifestParserUtils.ContentType;
     const audioMediaState = this.mediaStates_.get(ContentType.AUDIO);
     if (audioMediaState) {
       audioMediaState.lastInitSegmentReference = null;
-      this.forceClearBuffer_(audioMediaState);
+      if (clearBuffer) {
+        this.forceClearBuffer_(audioMediaState);
+      }
       this.abortOperations_(audioMediaState).catch(() => {});
     }
     const videoMediaState = this.mediaStates_.get(ContentType.VIDEO);
     if (videoMediaState) {
       videoMediaState.lastInitSegmentReference = null;
-      this.forceClearBuffer_(videoMediaState);
+      if (clearBuffer) {
+        this.forceClearBuffer_(videoMediaState);
+      }
       this.abortOperations_(videoMediaState).catch(() => {});
     }
     /**
@@ -2901,6 +2920,14 @@ shaka.media.StreamingEngine = class {
       streamsByType.set(ContentType.VIDEO, this.currentVariant_.video);
     }
     await this.playerInterface_.mediaSourceEngine.reset(streamsByType);
+    if (videoMediaState &&
+        !videoMediaState.performingUpdate && !videoMediaState.updateTimer) {
+      this.scheduleUpdate_(videoMediaState, 0);
+    }
+    if (audioMediaState &&
+          !audioMediaState.performingUpdate && !audioMediaState.updateTimer) {
+      this.scheduleUpdate_(audioMediaState, 0);
+    }
     return true;
   }
 

lib/mss/mss_parser.js

@@ -605,6 +605,7 @@ shaka.mss.MssParser = class {
       external: false,
       fastSwitching: false,
       fullMimeTypes: new Set(),
+      isAudioMuxedInVideo: false,
     };
 
     // This is specifically for text tracks.

lib/offline/indexeddb/v1_storage_cell.js

@@ -182,6 +182,7 @@ shaka.offline.indexeddb.V1StorageCell = class
       tilesLayout: undefined,
       external: false,
       fastSwitching: false,
+      isAudioMuxedInVideo: false,
     };
   }
 

lib/offline/indexeddb/v2_storage_cell.js

@@ -131,6 +131,7 @@ shaka.offline.indexeddb.V2StorageCell = class
       tilesLayout: undefined,
       external: false,
       fastSwitching: false,
+      isAudioMuxedInVideo: false,
     };
   }
 

lib/offline/manifest_converter.js

@@ -221,6 +221,7 @@ shaka.offline.ManifestConverter = class {
       fastSwitching: streamDB.fastSwitching,
       fullMimeTypes: new Set([shaka.util.MimeUtils.getFullType(
           streamDB.mimeType, streamDB.codecs)]),
+      isAudioMuxedInVideo: false,
     };
 
     return stream;