Resync libwebrtc to M115.0.5790.95

https://bugs.webkit.org/show_bug.cgi?id=259183
rdar://problem/112187110

Reviewed by Eric Carlson.

Cherry-pick webrtc patches that got merged in M115 branch: ad3838a9b5 ba943f71e6 f0d954f659 f5ffd45855 6603550a42 43670de877

* Source/ThirdParty/libwebrtc/Source/webrtc/api/video_codecs/video_codec.h:
* Source/ThirdParty/libwebrtc/Source/webrtc/call/rtp_payload_params.cc:
* Source/ThirdParty/libwebrtc/Source/webrtc/call/rtp_payload_params_unittest.cc:
* Source/ThirdParty/libwebrtc/Source/webrtc/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc:
* Source/ThirdParty/libwebrtc/Source/webrtc/pc/BUILD.gn:
* Source/ThirdParty/libwebrtc/Source/webrtc/pc/data_channel_controller.cc:
* Source/ThirdParty/libwebrtc/Source/webrtc/pc/peer_connection.cc:
* Source/ThirdParty/libwebrtc/Source/webrtc/pc/peer_connection.h:
* Source/ThirdParty/libwebrtc/Source/webrtc/pc/peer_connection_encodings_integrationtest.cc:
* Source/ThirdParty/libwebrtc/Source/webrtc/pc/peer_connection_integrationtest.cc:
* Source/ThirdParty/libwebrtc/Source/webrtc/pc/webrtc_sdp.cc:
* Source/ThirdParty/libwebrtc/Source/webrtc/pc/webrtc_sdp_unittest.cc:
* Source/ThirdParty/libwebrtc/Source/webrtc/video/adaptation/bitrate_constraint.cc:
* Source/ThirdParty/libwebrtc/Source/webrtc/video/adaptation/bitrate_constraint_unittest.cc:
* Source/ThirdParty/libwebrtc/Source/webrtc/video/adaptation/video_stream_encoder_resource_manager.cc:
* Source/ThirdParty/libwebrtc/Source/webrtc/video/adaptation/video_stream_encoder_resource_manager.h:
* Source/ThirdParty/libwebrtc/Source/webrtc/video/config/video_encoder_config.h:
* Source/ThirdParty/libwebrtc/Source/webrtc/video/video_stream_encoder.cc:

Canonical link: https://commits.webkit.org/266045@main

Author: youennf

Source/ThirdParty/libwebrtc/Source/webrtc/api/video_codecs/video_codec.h

@@ -148,6 +148,15 @@ class RTC_EXPORT VideoCodec {
   bool active;
 
   unsigned int qpMax;
+  // The actual number of simulcast streams. This is <= 1 in singlecast (it can
+  // be 0 in old code paths), but it is also 1 in the {active,inactive,inactive}
+  // "single RTP simulcast" use case and the legacy kSVC use case. In all other
+  // cases this is the same as the number of encodings (which may include
+  // inactive encodings). In other words:
+  // - `numberOfSimulcastStreams <= 1` in singlecast and singlecast-like setups
+  //   including legacy kSVC (encodings interpreted as spatial layers) or
+  //   standard kSVC (1 active encoding).
+  // - `numberOfSimulcastStreams > 1` in simulcast of 2+ active encodings.
   unsigned char numberOfSimulcastStreams;
   SimulcastStream simulcastStream[kMaxSimulcastStreams];
   SpatialLayer spatialLayers[kMaxSpatialLayers];

Source/ThirdParty/libwebrtc/Source/webrtc/call/rtp_payload_params.cc

@@ -621,23 +621,42 @@ void RtpPayloadParams::Vp9ToGeneric(const CodecSpecificInfoVP9& vp9_info,
     // Create the array only if it is ever used.
     last_vp9_frame_id_.resize(kPictureDiffLimit);
   }
-  if (vp9_header.inter_layer_predicted && spatial_index > 0) {
-    result.dependencies.push_back(
-        last_vp9_frame_id_[vp9_header.picture_id % kPictureDiffLimit]
-                          [spatial_index - 1]);
-  }
-  if (vp9_header.inter_pic_predicted) {
-    for (size_t i = 0; i < vp9_header.num_ref_pics; ++i) {
-      // picture_id is 15 bit number that wraps around. Though undeflow may
-      // produce picture that exceeds 2^15, it is ok because in this
-      // code block only last 7 bits of the picture_id are used.
-      uint16_t depend_on = vp9_header.picture_id - vp9_header.pid_diff[i];
+
+  if (vp9_header.flexible_mode) {
+    if (vp9_header.inter_layer_predicted && spatial_index > 0) {
       result.dependencies.push_back(
-          last_vp9_frame_id_[depend_on % kPictureDiffLimit][spatial_index]);
+          last_vp9_frame_id_[vp9_header.picture_id % kPictureDiffLimit]
+                            [spatial_index - 1]);
+    }
+    if (vp9_header.inter_pic_predicted) {
+      for (size_t i = 0; i < vp9_header.num_ref_pics; ++i) {
+        // picture_id is 15 bit number that wraps around. Though undeflow may
+        // produce picture that exceeds 2^15, it is ok because in this
+        // code block only last 7 bits of the picture_id are used.
+        uint16_t depend_on = vp9_header.picture_id - vp9_header.pid_diff[i];
+        result.dependencies.push_back(
+            last_vp9_frame_id_[depend_on % kPictureDiffLimit][spatial_index]);
+      }
+    }
+    last_vp9_frame_id_[vp9_header.picture_id % kPictureDiffLimit]
+                      [spatial_index] = shared_frame_id;
+  } else {
+    // Implementing general conversion logic for non-flexible mode requires some
+    // work and we will almost certainly never need it, so for now support only
+    // non-layerd streams.
+    if (spatial_index > 0 || temporal_index > 0) {
+      // Prefer to generate no generic layering than an inconsistent one.
+      rtp_video_header.generic.reset();
+      return;
+    }
+
+    if (vp9_header.inter_pic_predicted) {
+      // Since we only support non-scalable streams we only need to save the
+      // last frame id.
+      result.dependencies.push_back(last_vp9_frame_id_[0][0]);
     }
+    last_vp9_frame_id_[0][0] = shared_frame_id;
   }
-  last_vp9_frame_id_[vp9_header.picture_id % kPictureDiffLimit][spatial_index] =
-      shared_frame_id;
 
   result.active_decode_targets =
       ((uint32_t{1} << num_temporal_layers * num_active_spatial_layers) - 1);

Source/ThirdParty/libwebrtc/Source/webrtc/call/rtp_payload_params_unittest.cc

@@ -567,6 +567,7 @@ TEST(RtpPayloadParamsVp9ToGenericTest, NoScalability) {
   EncodedImage encoded_image;
   CodecSpecificInfo codec_info;
   codec_info.codecType = kVideoCodecVP9;
+  codec_info.codecSpecific.VP9.flexible_mode = true;
   codec_info.codecSpecific.VP9.num_spatial_layers = 1;
   codec_info.codecSpecific.VP9.temporal_idx = kNoTemporalIdx;
   codec_info.codecSpecific.VP9.first_frame_in_picture = true;
@@ -611,6 +612,55 @@ TEST(RtpPayloadParamsVp9ToGenericTest, NoScalability) {
   EXPECT_EQ(header.generic->chain_diffs[0], 3 - 1);
 }
 
+TEST(RtpPayloadParamsVp9ToGenericTest, NoScalabilityNonFlexibleMode) {
+  RtpPayloadState state;
+  RtpPayloadParams params(/*ssrc=*/123, &state, FieldTrialBasedConfig());
+
+  EncodedImage encoded_image;
+  CodecSpecificInfo codec_info;
+  codec_info.codecType = kVideoCodecVP9;
+  codec_info.codecSpecific.VP9.flexible_mode = false;
+  codec_info.codecSpecific.VP9.num_spatial_layers = 1;
+  codec_info.codecSpecific.VP9.temporal_idx = kNoTemporalIdx;
+  codec_info.codecSpecific.VP9.first_frame_in_picture = true;
+  codec_info.end_of_picture = true;
+
+  // Key frame.
+  encoded_image._frameType = VideoFrameType::kVideoFrameKey;
+  codec_info.codecSpecific.VP9.inter_pic_predicted = false;
+  RTPVideoHeader key_header =
+      params.GetRtpVideoHeader(encoded_image, &codec_info,
+                               /*shared_frame_id=*/1);
+
+  ASSERT_TRUE(key_header.generic);
+  EXPECT_EQ(key_header.generic->spatial_index, 0);
+  EXPECT_EQ(key_header.generic->temporal_index, 0);
+  EXPECT_EQ(key_header.generic->frame_id, 1);
+  ASSERT_THAT(key_header.generic->decode_target_indications, Not(IsEmpty()));
+  EXPECT_EQ(key_header.generic->decode_target_indications[0],
+            DecodeTargetIndication::kSwitch);
+  EXPECT_THAT(key_header.generic->dependencies, IsEmpty());
+  ASSERT_THAT(key_header.generic->chain_diffs, Not(IsEmpty()));
+  EXPECT_EQ(key_header.generic->chain_diffs[0], 0);
+
+  encoded_image._frameType = VideoFrameType::kVideoFrameDelta;
+  codec_info.codecSpecific.VP9.inter_pic_predicted = true;
+  RTPVideoHeader delta_header =
+      params.GetRtpVideoHeader(encoded_image, &codec_info,
+                               /*shared_frame_id=*/3);
+
+  ASSERT_TRUE(delta_header.generic);
+  EXPECT_EQ(delta_header.generic->spatial_index, 0);
+  EXPECT_EQ(delta_header.generic->temporal_index, 0);
+  EXPECT_EQ(delta_header.generic->frame_id, 3);
+  ASSERT_THAT(delta_header.generic->decode_target_indications, Not(IsEmpty()));
+  EXPECT_EQ(delta_header.generic->decode_target_indications[0],
+            DecodeTargetIndication::kSwitch);
+  EXPECT_THAT(delta_header.generic->dependencies, ElementsAre(1));
+  ASSERT_THAT(delta_header.generic->chain_diffs, Not(IsEmpty()));
+  EXPECT_EQ(delta_header.generic->chain_diffs[0], 3 - 1);
+}
+
 TEST(RtpPayloadParamsVp9ToGenericTest, TemporalScalabilityWith2Layers) {
   // Test with 2 temporal layers structure that is not used by webrtc:
   //    1---3   5
@@ -622,6 +672,7 @@ TEST(RtpPayloadParamsVp9ToGenericTest, TemporalScalabilityWith2Layers) {
   EncodedImage image;
   CodecSpecificInfo info;
   info.codecType = kVideoCodecVP9;
+  info.codecSpecific.VP9.flexible_mode = true;
   info.codecSpecific.VP9.num_spatial_layers = 1;
   info.codecSpecific.VP9.first_frame_in_picture = true;
   info.end_of_picture = true;
@@ -732,6 +783,7 @@ TEST(RtpPayloadParamsVp9ToGenericTest, TemporalScalabilityWith3Layers) {
   EncodedImage image;
   CodecSpecificInfo info;
   info.codecType = kVideoCodecVP9;
+  info.codecSpecific.VP9.flexible_mode = true;
   info.codecSpecific.VP9.num_spatial_layers = 1;
   info.codecSpecific.VP9.first_frame_in_picture = true;
   info.end_of_picture = true;
@@ -885,6 +937,7 @@ TEST(RtpPayloadParamsVp9ToGenericTest, SpatialScalabilityKSvc) {
   EncodedImage image;
   CodecSpecificInfo info;
   info.codecType = kVideoCodecVP9;
+  info.codecSpecific.VP9.flexible_mode = true;
   info.codecSpecific.VP9.num_spatial_layers = 2;
   info.codecSpecific.VP9.first_frame_in_picture = true;
 
@@ -993,6 +1046,7 @@ TEST(RtpPayloadParamsVp9ToGenericTest,
   EncodedImage image;
   CodecSpecificInfo info;
   info.codecType = kVideoCodecVP9;
+  info.codecSpecific.VP9.flexible_mode = true;
   info.codecSpecific.VP9.num_spatial_layers = 1;
   info.codecSpecific.VP9.first_frame_in_picture = true;
 

Source/ThirdParty/libwebrtc/Source/webrtc/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc

@@ -144,7 +144,6 @@ class SharedScreenCastStreamPrivate {
   uint32_t frame_rate_ = 60;
 
   bool use_damage_region_ = true;
-  DesktopRegion last_damage_region_;
 
   // Specifies whether the pipewire stream has been initialized with a request
   // to embed cursor into the captured frames.
@@ -159,9 +158,7 @@ class SharedScreenCastStreamPrivate {
   void ProcessBuffer(pw_buffer* buffer);
   bool ProcessMemFDBuffer(pw_buffer* buffer,
                           DesktopFrame& frame,
-                          const DesktopFrame* previous_frame,
-                          const DesktopVector& offset,
-                          bool effectively_new_frame);
+                          const DesktopVector& offset);
   bool ProcessDMABuffer(pw_buffer* buffer,
                         DesktopFrame& frame,
                         const DesktopVector& offset);
@@ -828,22 +825,17 @@ void SharedScreenCastStreamPrivate::ProcessBuffer(pw_buffer* buffer) {
     }
   }
 
-  bool effectively_new_frame = false;
   if (!queue_.current_frame() ||
       !queue_.current_frame()->size().equals(frame_size_)) {
     std::unique_ptr<DesktopFrame> frame(new BasicDesktopFrame(
         DesktopSize(frame_size_.width(), frame_size_.height())));
     queue_.ReplaceCurrentFrame(SharedDesktopFrame::Wrap(std::move(frame)));
-    effectively_new_frame = true;
   }
 
-  UpdateFrameUpdatedRegions(spa_buffer, *queue_.current_frame());
-
   bool bufferProcessed = false;
   if (spa_buffer->datas[0].type == SPA_DATA_MemFd) {
-    bufferProcessed = ProcessMemFDBuffer(buffer, *queue_.current_frame(),
-                                         queue_.previous_frame(), offset,
-                                         effectively_new_frame);
+    bufferProcessed =
+        ProcessMemFDBuffer(buffer, *queue_.current_frame(), offset);
   } else if (spa_buffer->datas[0].type == SPA_DATA_DmaBuf) {
     bufferProcessed = ProcessDMABuffer(buffer, *queue_.current_frame(), offset);
   }
@@ -870,6 +862,7 @@ void SharedScreenCastStreamPrivate::ProcessBuffer(pw_buffer* buffer) {
     observer_->OnDesktopFrameChanged();
   }
 
+  UpdateFrameUpdatedRegions(spa_buffer, *queue_.current_frame());
   queue_.current_frame()->set_may_contain_cursor(is_cursor_embedded_);
 
   if (callback_) {
@@ -885,9 +878,7 @@ RTC_NO_SANITIZE("cfi-icall")
 bool SharedScreenCastStreamPrivate::ProcessMemFDBuffer(
     pw_buffer* buffer,
     DesktopFrame& frame,
-    const DesktopFrame* previous_frame,
-    const DesktopVector& offset,
-    bool effectively_new_frame) {
+    const DesktopVector& offset) {
   spa_buffer* spa_buffer = buffer->buffer;
   ScopedBuf map;
   uint8_t* src = nullptr;
@@ -913,31 +904,9 @@ bool SharedScreenCastStreamPrivate::ProcessMemFDBuffer(
   uint8_t* updated_src =
       src + (src_stride * offset.y()) + (kBytesPerPixel * offset.x());
 
-  const int stride = src_stride - (kBytesPerPixel * offset.x());
-
-  if (effectively_new_frame || !previous_frame || !use_damage_region_ ||
-      damage_region_.is_empty()) {
-    frame.CopyPixelsFrom(
-        updated_src, stride,
-        DesktopRect::MakeWH(frame.size().width(), frame.size().height()));
-  } else {
-    for (DesktopRegion::Iterator it(last_damage_region_); !it.IsAtEnd();
-         it.Advance()) {
-      const DesktopRect& r = it.rect();
-      frame.CopyPixelsFrom(*previous_frame, r.top_left(), r);
-    }
-
-    for (DesktopRegion::Iterator it(damage_region_); !it.IsAtEnd();
-         it.Advance()) {
-      const auto& rect = it.rect();
-      frame.CopyPixelsFrom(src + rect.top() * stride +
-                               rect.left() * DesktopFrame::kBytesPerPixel,
-                           stride,
-                           DesktopRect::MakeXYWH(rect.left(), rect.top(),
-                                                 rect.width(), rect.height()));
-    }
-  }
-  last_damage_region_ = damage_region_;
+  frame.CopyPixelsFrom(
+      updated_src, (src_stride - (kBytesPerPixel * offset.x())),
+      DesktopRect::MakeWH(frame.size().width(), frame.size().height()));
 
   return true;
 }

Source/ThirdParty/libwebrtc/Source/webrtc/pc/BUILD.gn

@@ -2430,6 +2430,7 @@ if (rtc_include_tests && !build_with_chromium) {
       "../api/transport:field_trial_based_config",
       "../api/transport:sctp_transport_factory_interface",
       "../api/transport/rtp:rtp_source",
+      "../api/units:data_rate",
       "../api/units:time_delta",
       "../api/units:timestamp",
       "../api/video:builtin_video_bitrate_allocator_factory",

Source/ThirdParty/libwebrtc/Source/webrtc/pc/data_channel_controller.cc

@@ -70,15 +70,19 @@ void DataChannelController::OnChannelStateChanged(
     SctpDataChannel* channel,
     DataChannelInterface::DataState state) {
   RTC_DCHECK_RUN_ON(network_thread());
+
+  // Stash away the internal id here in case `OnSctpDataChannelClosed` ends up
+  // releasing the last reference to the channel.
+  const int channel_id = channel->internal_id();
+
   if (state == DataChannelInterface::DataState::kClosed)
     OnSctpDataChannelClosed(channel);
 
   DataChannelUsage channel_usage = sctp_data_channels_n_.empty()
                                        ? DataChannelUsage::kHaveBeenUsed
                                        : DataChannelUsage::kInUse;
   signaling_thread()->PostTask(SafeTask(
-      signaling_safety_.flag(), [this, channel_id = channel->internal_id(),
-                                 state = state, channel_usage] {
+      signaling_safety_.flag(), [this, channel_id, state, channel_usage] {
         RTC_DCHECK_RUN_ON(signaling_thread());
         channel_usage_ = channel_usage;
         pc_->OnSctpDataChannelStateChanged(channel_id, state);

Source/ThirdParty/libwebrtc/Source/webrtc/pc/peer_connection.cc

@@ -739,9 +739,6 @@ JsepTransportController* PeerConnection::InitializeTransportController_n(
   transport_controller_->SubscribeIceConnectionState(
       [this](cricket::IceConnectionState s) {
         RTC_DCHECK_RUN_ON(network_thread());
-        if (s == cricket::kIceConnectionConnected) {
-          ReportTransportStats();
-        }
         signaling_thread()->PostTask(
             SafeTask(signaling_thread_safety_.flag(), [this, s]() {
               RTC_DCHECK_RUN_ON(signaling_thread());
@@ -2405,6 +2402,20 @@ void PeerConnection::OnTransportControllerConnectionState(
     case cricket::kIceConnectionConnected:
       RTC_LOG(LS_INFO) << "Changing to ICE connected state because "
                           "all transports are writable.";
+      {
+        std::vector<RtpTransceiverProxyRefPtr> transceivers;
+        if (ConfiguredForMedia()) {
+          transceivers = rtp_manager()->transceivers()->List();
+        }
+
+        network_thread()->PostTask(
+            SafeTask(network_thread_safety_,
+                     [this, transceivers = std::move(transceivers)] {
+                       RTC_DCHECK_RUN_ON(network_thread());
+                       ReportTransportStats(std::move(transceivers));
+                     }));
+      }
+
       SetIceConnectionState(PeerConnectionInterface::kIceConnectionConnected);
       NoteUsageEvent(UsageEvent::ICE_STATE_CONNECTED);
       break;
@@ -2748,20 +2759,18 @@ void PeerConnection::OnTransportControllerGatheringState(
 }
 
 // Runs on network_thread().
-void PeerConnection::ReportTransportStats() {
+void PeerConnection::ReportTransportStats(
+    std::vector<RtpTransceiverProxyRefPtr> transceivers) {
   TRACE_EVENT0("webrtc", "PeerConnection::ReportTransportStats");
   rtc::Thread::ScopedDisallowBlockingCalls no_blocking_calls;
   std::map<std::string, std::set<cricket::MediaType>>
       media_types_by_transport_name;
-  if (ConfiguredForMedia()) {
-    for (const auto& transceiver :
-         rtp_manager()->transceivers()->UnsafeList()) {
-      if (transceiver->internal()->channel()) {
-        std::string transport_name(
-            transceiver->internal()->channel()->transport_name());
-        media_types_by_transport_name[transport_name].insert(
-            transceiver->media_type());
-      }
+  for (const auto& transceiver : transceivers) {
+    if (transceiver->internal()->channel()) {
+      std::string transport_name(
+          transceiver->internal()->channel()->transport_name());
+      media_types_by_transport_name[transport_name].insert(
+          transceiver->media_type());
     }
   }
 

Source/ThirdParty/libwebrtc/Source/webrtc/pc/peer_connection.h

@@ -567,7 +567,8 @@ class PeerConnection : public PeerConnectionInternal,
 
   // Invoked when TransportController connection completion is signaled.
   // Reports stats for all transports in use.
-  void ReportTransportStats() RTC_RUN_ON(network_thread());
+  void ReportTransportStats(std::vector<RtpTransceiverProxyRefPtr> transceivers)
+      RTC_RUN_ON(network_thread());
 
   // Gather the usage of IPv4/IPv6 as best connection.
   static void ReportBestConnectionState(const cricket::TransportStats& stats);