Add state setters to MediaSourceAttachment (#3801)

In preparation for cross-thread MediaSource objects, a small amount of
state and associated setters are being added to the
MediaSourceAttachment interface. This will allow for a push model,
instead of a pull model, for state such as recent media timestamps, and
the error status of the HTMLMediaElement.

Usage of these setters is gated through the
`"MediaElement.EnableUsingMediaSourceAttachmentMethods"` H5VCC flag.
This flag needs to be coordinated between a `HTMLMediaElement`, the
attached `MediaSource`, and all `SourceBuffers` used. As such, the value
of the flag when the `HTMLMediaElement` was created is used to
coordinate the state of the rest of the related objects. This prevents a
flag mismatch, such as if the H5VCC value is updated after the
`HTMLMediaElement` has been created but before the `MediaSource` is
created.

This is based off of the following Chromium changes:

*  https://chromium-review.googlesource.com/c/chromium/src/+/2391934
*  https://chromium-review.googlesource.com/c/chromium/src/+/2401808

b/338425449

cobalt/dom/html_media_element.cc

@@ -431,9 +431,11 @@ void HTMLMediaElement::set_current_time(
     LOG(ERROR) << "invalid state error";
     web::DOMException::Raise(web::DOMException::kInvalidStateErr,
                              exception_state);
-    return;
+  } else {
+    Seek(time);
   }
-  Seek(time);
+
+  ReportCurrentTimeToMediaSource();
 }
 
 double HTMLMediaElement::duration() const {
@@ -816,7 +818,7 @@ void HTMLMediaElement::PrepareForLoad() {
   set_playback_rate(default_playback_rate());
 
   // 6 - Set the error attribute to null and the autoplaying flag to true.
-  error_ = NULL;
+  SetError(NULL);
   autoplaying_ = true;
 
   // 7 - Invoke the media element's resource selection algorithm.
@@ -1050,8 +1052,8 @@ void HTMLMediaElement::NoneSupported(const std::string& message) {
 
   // 6.1 - Set the error attribute to a new MediaError object whose code
   // attribute is set to MEDIA_ERR_SRC_NOT_SUPPORTED.
-  error_ = new MediaError(MediaError::kMediaErrSrcNotSupported,
-                          message.empty() ? "Source not supported." : message);
+  SetError(new MediaError(MediaError::kMediaErrSrcNotSupported,
+                          message.empty() ? "Source not supported." : message));
   // 6.2 - Forget the media element's media-resource-specific text tracks.
 
   // 6.3 - Set the element's networkState attribute to the kNetworkNoSource
@@ -1129,6 +1131,10 @@ void HTMLMediaElement::OnPlaybackProgressTimer() {
   }
 
   ScheduleTimeupdateEvent(true);
+
+  // The playback progress timer is used here to provide a steady clock that
+  // allows the attached MediaSource to have access to a recent media time.
+  ReportCurrentTimeToMediaSource();
 }
 
 void HTMLMediaElement::StartPlaybackProgressTimer() {
@@ -1511,6 +1517,8 @@ void HTMLMediaElement::UpdatePlayState() {
       AddPlayedRange(last_seek_time_, time);
     }
   }
+
+  ReportCurrentTimeToMediaSource();
 }
 
 bool HTMLMediaElement::PotentiallyPlaying() const {
@@ -1575,6 +1583,32 @@ void HTMLMediaElement::ConfigureMediaControls() {
   DLOG_IF(WARNING, controls_) << "media control is not supported";
 }
 
+void HTMLMediaElement::ReportCurrentTimeToMediaSource() {
+  if (!is_using_media_source_attachment_methods_) {
+    return;
+  }
+
+  if (!media_source_attachment_) {
+    return;
+  }
+
+  media_source_attachment_->OnElementTimeUpdate(current_time(NULL));
+}
+
+void HTMLMediaElement::SetError(scoped_refptr<MediaError> error) {
+  error_ = error;
+
+  if (!is_using_media_source_attachment_methods_) {
+    return;
+  }
+
+  if (!error || !media_source_attachment_) {
+    return;
+  }
+
+  media_source_attachment_->OnElementError();
+}
+
 void HTMLMediaElement::MediaEngineError(scoped_refptr<MediaError> error) {
   if (error->message().empty()) {
     LOG(WARNING) << "HTMLMediaElement::MediaEngineError " << error->code();
@@ -1589,7 +1623,7 @@ void HTMLMediaElement::MediaEngineError(scoped_refptr<MediaError> error) {
 
   // 2 - Set the error attribute to a new MediaError object whose code attribute
   // is set to MEDIA_ERR_NETWORK/MEDIA_ERR_DECODE.
-  error_ = error;
+  SetError(error);
 
   // 3 - Queue a task to fire a simple event named error at the media element.
   ScheduleOwnEvent(base::Tokens::error());

cobalt/dom/html_media_element.h

@@ -229,7 +229,13 @@ class HTMLMediaElement : public HTMLElement,
 
   void ConfigureMediaControls();
 
+  // Pushes the current media time to the attached MediaSourceAttachment to
+  // avoid having to asynchronously pull media time from a cross-thread
+  // MediaSource object.
+  void ReportCurrentTimeToMediaSource();
+
   // Error report
+  void SetError(scoped_refptr<MediaError> error);
   void MediaEngineError(scoped_refptr<MediaError> error);
 
   // WebMediaPlayerClient methods
@@ -313,6 +319,7 @@ class HTMLMediaElement : public HTMLElement,
   // Time has not changed since sending an "ended" event.
   bool sent_end_event_;
 
+  // See SetError().
   scoped_refptr<MediaError> error_;
 
   // Helper object to reduce the image capacity while a video is playing.

cobalt/dom/media_source.cc

@@ -145,17 +145,6 @@ bool IsMediaElementUsingMediaSourceBufferedRangeEnabled(
       .value_or(false);
 }
 
-// If this function returns true, MediaSource will proxy calls to the
-// attached HTMLMediaElement object through the MediaSourceAttachment interface
-// instead of directly calling against the HTMLMediaElement object.
-// The default value is false.
-bool IsMediaElementUsingMediaSourceAttachmentMethodsEnabled(
-    web::EnvironmentSettings* settings) {
-  return GetMediaSettings(settings)
-      .IsMediaElementUsingMediaSourceAttachmentMethodsEnabled()
-      .value_or(false);
-}
-
 }  // namespace
 
 MediaSource::MediaSource(script::EnvironmentSettings* settings)
@@ -170,9 +159,6 @@ MediaSource::MediaSource(script::EnvironmentSettings* settings)
       chunk_demuxer_(NULL),
       ready_state_(kMediaSourceReadyStateClosed),
       ALLOW_THIS_IN_INITIALIZER_LIST(event_queue_(this)),
-      is_using_media_source_attachment_methods_(
-          IsMediaElementUsingMediaSourceAttachmentMethodsEnabled(
-              environment_settings())),
       source_buffers_(new SourceBufferList(settings, &event_queue_)),
       active_source_buffers_(new SourceBufferList(settings, &event_queue_)),
       live_seekable_range_(new TimeRanges) {
@@ -294,7 +280,8 @@ scoped_refptr<SourceBuffer> MediaSource::AddSourceBuffer(
   switch (status) {
     case ChunkDemuxer::kOk:
       source_buffer =
-          new SourceBuffer(settings, guid, this, chunk_demuxer_, &event_queue_);
+          new SourceBuffer(settings, guid, this, chunk_demuxer_, &event_queue_,
+                           is_using_media_source_attachment_methods_);
       break;
     case ChunkDemuxer::kNotSupported:
       web::DOMException::Raise(web::DOMException::kNotSupportedErr,
@@ -439,6 +426,8 @@ bool MediaSource::IsTypeSupported(script::EnvironmentSettings* settings,
 
 bool MediaSource::StartAttachingToMediaElement(
     HTMLMediaElement* media_element) {
+  is_using_media_source_attachment_methods_ = false;
+
   if (attached_element_) {
     return false;
   }
@@ -472,6 +461,8 @@ bool MediaSource::StartAttachingToMediaElement(
 
 bool MediaSource::StartAttachingToMediaElement(
     MediaSourceAttachmentSupplement* media_source_attachment) {
+  is_using_media_source_attachment_methods_ = true;
+
   if (media_source_attachment_) {
     return false;
   }
@@ -685,10 +676,12 @@ void MediaSource::SetSourceBufferActive(SourceBuffer* source_buffer,
 }
 
 HTMLMediaElement* MediaSource::GetMediaElement() const {
+  DCHECK(!is_using_media_source_attachment_methods_);
   return attached_element_;
 }
 
 MediaSourceAttachmentSupplement* MediaSource::GetMediaSourceAttachment() const {
+  DCHECK(is_using_media_source_attachment_methods_);
   return media_source_attachment_;
 }
 

cobalt/dom/media_source.h

@@ -170,7 +170,7 @@ class MediaSource : public web::EventTarget {
   MediaSourceReadyState ready_state_;
   EventQueue event_queue_;
   // TODO(b/338425449): Remove direct references to HTMLMediaElement.
-  const bool is_using_media_source_attachment_methods_;
+  bool is_using_media_source_attachment_methods_ = false;
   base::WeakPtr<HTMLMediaElement> attached_element_;
   base::WeakPtr<MediaSourceAttachmentSupplement> media_source_attachment_;
 

cobalt/dom/media_source_attachment.h

@@ -71,6 +71,22 @@ class MediaSourceAttachment
   // flag MediaElement.EnableUsingMediaSourceAttachmentMethods is disabled.
   virtual scoped_refptr<MediaSource> media_source() const = 0;
 
+  // Provide state updates to the MediaSource that are necessary for its
+  // operation. These are pushed rather than pulled to reduce complexity and
+  // latency, especially when the MediaSource is in a Worker context.
+  // OnElementTimeUpdate() gives the MediaSource a notion of the recent media
+  // element currentTime so that it can more effectively prevent evicting
+  // buffered media near to playback and/or seek target time in its heuristic.
+  // Alternatives such as pumping this via the media pipeline are insufficient,
+  // as the media pipeline may not be aware of overrides to the playback start
+  // position.
+  virtual void OnElementTimeUpdate(double time) = 0;
+
+  // Needed as a precondition in the Prepare Append algorithm, OnElementError()
+  // lets the MediaSource know if the attached media element has transitioned to
+  // having an error.
+  virtual void OnElementError() = 0;
+
  private:
   friend class base::RefCountedThreadSafe<MediaSourceAttachment>;
 

cobalt/dom/same_thread_media_source_attachment.cc

@@ -34,7 +34,9 @@ namespace dom {
 SameThreadMediaSourceAttachment::SameThreadMediaSourceAttachment(
     scoped_refptr<MediaSource> media_source)
     : media_source_(media_source),
-      task_runner_(base::SequencedTaskRunner::GetCurrentDefault()) {}
+      task_runner_(base::SequencedTaskRunner::GetCurrentDefault()),
+      recent_element_time_(0.0),
+      element_has_error_(false) {}
 
 void SameThreadMediaSourceAttachment::TraceMembers(script::Tracer* tracer) {
   DCHECK_EQ(task_runner_, base::SequencedTaskRunner::GetCurrentDefault());
@@ -108,14 +110,23 @@ double SameThreadMediaSourceAttachment::GetRecentMediaTime() {
   DCHECK_EQ(task_runner_, base::SequencedTaskRunner::GetCurrentDefault());
   DCHECK(attached_element_);
 
-  return attached_element_->current_time(NULL);
+  double result = attached_element_->current_time(NULL);
+
+  DVLOG(2) << __func__ << ": recent time=" << recent_element_time_
+           << ", actual current time=" << result;
+
+  return result;
 }
 
 bool SameThreadMediaSourceAttachment::GetElementError() {
   DCHECK_EQ(task_runner_, base::SequencedTaskRunner::GetCurrentDefault());
   DCHECK(attached_element_);
 
-  return static_cast<bool>(attached_element_->error());
+  bool result = static_cast<bool>(attached_element_->error());
+
+  DCHECK_EQ(result, element_has_error_);
+
+  return result;
 }
 
 scoped_refptr<AudioTrackList>
@@ -136,5 +147,16 @@ SameThreadMediaSourceAttachment::CreateVideoTrackList(
   return base::MakeRefCounted<VideoTrackList>(settings, attached_element_);
 }
 
+void SameThreadMediaSourceAttachment::OnElementTimeUpdate(double time) {
+  recent_element_time_ = time;
+}
+
+void SameThreadMediaSourceAttachment::OnElementError() {
+  DCHECK(!element_has_error_)
+      << "At most one transition to element error per attachment is expected";
+
+  element_has_error_ = true;
+}
+
 }  // namespace dom
 }  // namespace cobalt

cobalt/dom/same_thread_media_source_attachment.h

@@ -51,6 +51,9 @@ class SameThreadMediaSourceAttachment : public MediaSourceAttachmentSupplement {
   scoped_refptr<TimeRanges> GetBufferedRange() const override;
   MediaSourceReadyState GetReadyState() const override;
 
+  void OnElementTimeUpdate(double time) override;
+  void OnElementError() override;
+
   // MediaSourceAttachmentSupplement
   void NotifyDurationChanged(double duration) override;
   bool HasMaxVideoCapabilities() const override;
@@ -84,6 +87,9 @@ class SameThreadMediaSourceAttachment : public MediaSourceAttachmentSupplement {
   // Used to ensure all calls are made on the thread that created this object.
   base::SequencedTaskRunner* task_runner_;
 
+  double recent_element_time_;  // See OnElementTimeUpdate().
+  bool element_has_error_;      // See OnElementError().
+
   DISALLOW_COPY_AND_ASSIGN(SameThreadMediaSourceAttachment);
 };
 

cobalt/dom/source_buffer.cc

@@ -129,17 +129,6 @@ bool IsAvoidCopyingArrayBufferEnabled(web::EnvironmentSettings* settings) {
   return media_settings.IsAvoidCopyingArrayBufferEnabled().value_or(false);
 }
 
-// If this function returns true, MediaSource will proxy calls to the
-// attached HTMLMediaElement object through the MediaSourceAttachment interface
-// instead of directly calling against the HTMLMediaElement object.
-// The default value is false.
-bool IsMediaElementUsingMediaSourceAttachmentMethodsEnabled(
-    web::EnvironmentSettings* settings) {
-  return GetMediaSettings(settings)
-      .IsMediaElementUsingMediaSourceAttachmentMethodsEnabled()
-      .value_or(false);
-}
-
 }  // namespace
 
 SourceBuffer::OnInitSegmentReceivedHelper::OnInitSegmentReceivedHelper(
@@ -170,7 +159,8 @@ void SourceBuffer::OnInitSegmentReceivedHelper::TryToRunOnInitSegmentReceived(
 
 SourceBuffer::SourceBuffer(script::EnvironmentSettings* settings,
                            const std::string& id, MediaSource* media_source,
-                           ChunkDemuxer* chunk_demuxer, EventQueue* event_queue)
+                           ChunkDemuxer* chunk_demuxer, EventQueue* event_queue,
+                           const bool is_using_media_source_attachment_methods)
     : web::EventTarget(settings),
       on_init_segment_received_helper_(new OnInitSegmentReceivedHelper(this)),
       id_(id),
@@ -181,8 +171,7 @@ SourceBuffer::SourceBuffer(script::EnvironmentSettings* settings,
       chunk_demuxer_(chunk_demuxer),
       event_queue_(event_queue),
       is_using_media_source_attachment_methods_(
-          IsMediaElementUsingMediaSourceAttachmentMethodsEnabled(
-              environment_settings())),
+          is_using_media_source_attachment_methods),
       audio_tracks_(
           is_using_media_source_attachment_methods_
               ? media_source->GetMediaSourceAttachment()->CreateAudioTrackList(

cobalt/dom/source_buffer.h

@@ -90,9 +90,11 @@ class SourceBuffer : public web::EventTarget {
 
   // Custom, not in any spec.
   //
+  // TODO(b/338425449): Remove is_using_media_source_attachment_methods.
   SourceBuffer(script::EnvironmentSettings* settings, const std::string& id,
                MediaSource* media_source, ChunkDemuxer* chunk_demuxer,
-               EventQueue* event_queue);
+               EventQueue* event_queue,
+               const bool is_using_media_source_attachment_methods);
 
   // Web API: SourceBuffer
   //