[GStreamer][LibWebRTC] Timestamp handling improvements in video decoder factory

https://bugs.webkit.org/show_bug.cgi?id=273757

Reviewed by Xabier Rodriguez-Calvar.

The buffers injected into the parser pipeline are now timestamped by appsrc. The RTP timestamps are
required by the LibWebRTC generic decoder in order to match parsed frames with input buffers, so
they are attached on each buffers using a reference timestamp meta.

* Source/WebCore/platform/mediastream/libwebrtc/gstreamer/GStreamerVideoDecoderFactory.cpp:
(WebCore::GStreamerWebRTCVideoDecoder::GStreamerWebRTCVideoDecoder):
(WebCore::GStreamerWebRTCVideoDecoder::pullSample):
* Source/WebCore/platform/mediastream/libwebrtc/gstreamer/GStreamerVideoFrameLibWebRTC.cpp:
(WebCore::ensureDebugCategoryIsRegistered):
(WebCore::convertGStreamerSampleToLibWebRTCVideoFrame):
(WebCore::GStreamerVideoFrameLibWebRTC::ToI420):
* Source/WebCore/platform/mediastream/libwebrtc/gstreamer/GStreamerVideoFrameLibWebRTC.h:

Canonical link: https://commits.webkit.org/278447@main

Author: philn

Source/WebCore/platform/mediastream/libwebrtc/gstreamer/GStreamerVideoDecoderFactory.cpp

@@ -20,7 +20,7 @@
 
 #include "config.h"
 
-#if ENABLE(VIDEO) && ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER)
+#if USE(LIBWEBRTC) && USE(GSTREAMER)
 #include "GStreamerVideoDecoderFactory.h"
 
 #include "GStreamerQuirks.h"
@@ -45,22 +45,15 @@ GST_DEBUG_CATEGORY(webkit_webrtcdec_debug);
 
 namespace WebCore {
 
-typedef struct {
-    uint64_t timestamp;
-    int64_t renderTimeMs;
-} InputTimestamps;
-
 class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {
 public:
     GStreamerWebRTCVideoDecoder()
-        : m_pictureId(0)
-        , m_width(0)
+        : m_width(0)
         , m_height(0)
         , m_requireParse(false)
         , m_needsKeyframe(true)
-        , m_firstBufferPts(GST_CLOCK_TIME_NONE)
-        , m_firstBufferDts(GST_CLOCK_TIME_NONE)
     {
+        m_rtpTimestampCaps = adoptGRef(gst_caps_new_empty_simple("timestamp/x-rtp"));
     }
 
     static void decodebinPadAddedCb(GstElement*, GstPad* srcpad, GstPad* sinkpad)
@@ -103,6 +96,7 @@ class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {
     bool Configure(const webrtc::VideoDecoder::Settings& codecSettings) override
     {
         m_src = makeElement("appsrc");
+        g_object_set(m_src, "is-live", TRUE, "do-timestamp", TRUE, "max-buffers", 2, "max-bytes", 0, nullptr);
 
         GRefPtr<GstCaps> caps = nullptr;
         auto capsfilter = CreateFilter();
@@ -154,8 +148,7 @@ class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {
 
         m_sink = makeElement("appsink");
         gst_app_sink_set_emit_signals(GST_APP_SINK(m_sink), true);
-        // This is an decoder, everything should happen as fast as possible and not
-        // be synced on the clock.
+        // This is a decoder, everything should happen as fast as possible and not be synced on the clock.
         g_object_set(m_sink, "sync", false, nullptr);
 
         gst_bin_add_many(GST_BIN(pipeline()), m_src, decoder, capsfilter, m_sink, nullptr);
@@ -204,9 +197,7 @@ class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {
         return WEBRTC_VIDEO_CODEC_OK;
     }
 
-    int32_t Decode(const webrtc::EncodedImage& inputImage,
-        bool,
-        int64_t renderTimeMs) override
+    int32_t Decode(const webrtc::EncodedImage& inputImage, int64_t) override
     {
         if (m_needsKeyframe) {
             if (inputImage._frameType != webrtc::VideoFrameType::kVideoFrameKey) {
@@ -218,33 +209,15 @@ class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {
 
         if (!m_src) {
             GST_ERROR("No source set, can't decode.");
-
             return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
         }
 
-        // No renderTime provided, probably some issue with the WebRTC clock. Use a monotonically
-        // incrementing counter instead.
-        static int64_t s_forgedRenderTime { 0 };
-        if (!renderTimeMs) {
-            renderTimeMs = s_forgedRenderTime;
-            s_forgedRenderTime += 30 * GST_MSECOND;
-        }
+        // FIXME: Use a GstBufferPool.
+        GST_TRACE_OBJECT(pipeline(), "Pushing encoded image with RTP timestamp %u", inputImage.RtpTimestamp());
+        auto buffer = adoptGRef(gstBufferNewWrappedFast(fastMemDup(inputImage.data(), inputImage.size()), inputImage.size()));
 
-        if (!GST_CLOCK_TIME_IS_VALID(m_firstBufferPts)) {
-            GRefPtr<GstPad> srcpad = adoptGRef(gst_element_get_static_pad(m_src, "src"));
-            m_firstBufferPts = (static_cast<guint64>(renderTimeMs)) * GST_MSECOND;
-            m_firstBufferDts = (static_cast<guint64>(inputImage.RtpTimestamp())) * GST_MSECOND;
-        }
+        gst_buffer_add_reference_timestamp_meta(buffer.get(), m_rtpTimestampCaps.get(), inputImage.RtpTimestamp(), GST_CLOCK_TIME_NONE);
 
-        // FIXME- Use a GstBufferPool.
-        auto buffer = adoptGRef(gstBufferNewWrappedFast(fastMemDup(inputImage.data(), inputImage.size()),
-            inputImage.size()));
-        GST_BUFFER_DTS(buffer.get()) = (static_cast<guint64>(inputImage.RtpTimestamp()) * GST_MSECOND) - m_firstBufferDts;
-        GST_BUFFER_PTS(buffer.get()) = (static_cast<guint64>(renderTimeMs) * GST_MSECOND) - m_firstBufferPts;
-        InputTimestamps timestamps = { inputImage.RtpTimestamp(), renderTimeMs };
-        m_dtsPtsMap[GST_BUFFER_PTS(buffer.get())] = timestamps;
-
-        GST_LOG_OBJECT(pipeline(), "%" G_GINT64_FORMAT " Decoding: %" GST_PTR_FORMAT, renderTimeMs, buffer.get());
         auto sample = adoptGRef(gst_sample_new(buffer.get(), GetCapsForFrame(inputImage), nullptr, nullptr));
         switch (gst_app_src_push_sample(GST_APP_SRC(m_src), sample.get())) {
         case GST_FLOW_OK:
@@ -266,21 +239,11 @@ class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {
             return WEBRTC_VIDEO_CODEC_OK;
         }
         auto buffer = gst_sample_get_buffer(sample.get());
-
-        // Make sure that the frame.timestamp == previsouly input_frame._timeStamp
-        // as it is required by the VideoDecoder baseclass.
-        auto timestamps = m_dtsPtsMap[GST_BUFFER_PTS(buffer)];
-        m_dtsPtsMap.erase(GST_BUFFER_PTS(buffer));
-
-        auto frame(convertGStreamerSampleToLibWebRTCVideoFrame(WTFMove(sample), webrtc::kVideoRotation_0,
-            timestamps.timestamp, timestamps.renderTimeMs));
-
-        GST_BUFFER_DTS(buffer) = GST_CLOCK_TIME_NONE;
-        GST_LOG_OBJECT(pipeline(), "Output decoded frame! %d -> %" GST_PTR_FORMAT,
-            frame->timestamp(), buffer);
-
-        m_imageReadyCb->Decoded(*frame.get(), absl::optional<int32_t>(), absl::optional<uint8_t>());
-
+        auto meta = gst_buffer_get_reference_timestamp_meta(buffer, m_rtpTimestampCaps.get());
+        RELEASE_ASSERT(meta);
+        auto frame = convertGStreamerSampleToLibWebRTCVideoFrame(WTFMove(sample), meta->timestamp);
+        GST_TRACE_OBJECT(pipeline(), "Pulled video frame with RTP timestamp %u from %" GST_PTR_FORMAT, static_cast<uint32_t>(meta->timestamp), buffer);
+        m_imageReadyCb->Decoded(frame);
         return WEBRTC_VIDEO_CODEC_OK;
     }
 
@@ -326,7 +289,6 @@ class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {
 
 protected:
     GRefPtr<GstCaps> m_caps;
-    gint m_pictureId;
     gint m_width;
     gint m_height;
     bool m_requireParse = false;
@@ -339,9 +301,7 @@ class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {
 
     webrtc::DecodedImageCallback* m_imageReadyCb;
 
-    StdMap<GstClockTime, InputTimestamps> m_dtsPtsMap;
-    GstClockTime m_firstBufferPts;
-    GstClockTime m_firstBufferDts;
+    GRefPtr<GstCaps> m_rtpTimestampCaps;
 };
 
 class H264Decoder : public GStreamerWebRTCVideoDecoder {
@@ -474,5 +434,9 @@ std::vector<webrtc::SdpVideoFormat> GStreamerVideoDecoderFactory::GetSupportedFo
 
     return formats;
 }
-}
-#endif
+
+#undef GST_CAT_DEFAULT
+
+} // namespace WebCore
+
+#endif // USE(LIBWEBRTC) && USE(GSTREAMER)

Source/WebCore/platform/mediastream/libwebrtc/gstreamer/GStreamerVideoFrameLibWebRTC.cpp

@@ -27,6 +27,17 @@
 
 namespace WebCore {
 
+GST_DEBUG_CATEGORY(webkit_libwebrtc_video_frame_debug);
+#define GST_CAT_DEFAULT webkit_libwebrtc_video_frame_debug
+
+static void ensureDebugCategoryIsRegistered()
+{
+    static std::once_flag debugRegisteredFlag;
+    std::call_once(debugRegisteredFlag, [] {
+        GST_DEBUG_CATEGORY_INIT(webkit_libwebrtc_video_frame_debug, "webkitlibwebrtcvideoframe", 0, "WebKit LibWebRTC Video Frame");
+    });
+}
+
 GRefPtr<GstSample> convertLibWebRTCVideoFrameToGStreamerSample(const webrtc::VideoFrame& frame)
 {
     RELEASE_ASSERT(frame.video_frame_buffer()->type() != webrtc::VideoFrameBuffer::Type::kNative);
@@ -56,10 +67,15 @@ GRefPtr<GstSample> convertLibWebRTCVideoFrameToGStreamerSample(const webrtc::Vid
     return sample;
 }
 
-std::unique_ptr<webrtc::VideoFrame> convertGStreamerSampleToLibWebRTCVideoFrame(GRefPtr<GstSample>&& sample, webrtc::VideoRotation rotation, int64_t timestamp, int64_t renderTimeMs)
+webrtc::VideoFrame convertGStreamerSampleToLibWebRTCVideoFrame(GRefPtr<GstSample>&& sample, uint32_t rtpTimestamp)
 {
-    auto frameBuffer(GStreamerVideoFrameLibWebRTC::create(WTFMove(sample)));
-    return std::unique_ptr<webrtc::VideoFrame>(new webrtc::VideoFrame(WTFMove(frameBuffer), timestamp, renderTimeMs, rotation));
+    webrtc::VideoFrame::Builder builder;
+    auto buffer = gst_sample_get_buffer(sample.get());
+    auto pts = GST_BUFFER_PTS(buffer);
+    return builder.set_video_frame_buffer(GStreamerVideoFrameLibWebRTC::create(WTFMove(sample)))
+        .set_timestamp_rtp(rtpTimestamp)
+        .set_timestamp_us(pts)
+        .build();
 }
 
 rtc::scoped_refptr<webrtc::VideoFrameBuffer> GStreamerVideoFrameLibWebRTC::create(GRefPtr<GstSample>&& sample)
@@ -74,6 +90,7 @@ rtc::scoped_refptr<webrtc::VideoFrameBuffer> GStreamerVideoFrameLibWebRTC::creat
 
 rtc::scoped_refptr<webrtc::I420BufferInterface> GStreamerVideoFrameLibWebRTC::ToI420()
 {
+    ensureDebugCategoryIsRegistered();
     GstMappedFrame inFrame(m_sample, GST_MAP_READ);
     if (!inFrame) {
         GST_WARNING("Could not map input frame");
@@ -111,4 +128,6 @@ rtc::scoped_refptr<webrtc::I420BufferInterface> GStreamerVideoFrameLibWebRTC::To
 
 }
 
-#endif // USE(LIBWEBRTC)
+#undef GST_CAT_DEFAULT
+
+#endif // USE(LIBWEBRTC) && USE(GSTREAMER)

Source/WebCore/platform/mediastream/libwebrtc/gstreamer/GStreamerVideoFrameLibWebRTC.h

@@ -33,7 +33,7 @@ namespace WebCore {
 
 WARN_UNUSED_RETURN GRefPtr<GstSample> convertLibWebRTCVideoFrameToGStreamerSample(const webrtc::VideoFrame&);
 
-std::unique_ptr<webrtc::VideoFrame> convertGStreamerSampleToLibWebRTCVideoFrame(GRefPtr<GstSample>&&, webrtc::VideoRotation, int64_t timestamp, int64_t renderTimeMs);
+webrtc::VideoFrame convertGStreamerSampleToLibWebRTCVideoFrame(GRefPtr<GstSample>&&, uint32_t rtpTimestamp);
 
 class GStreamerVideoFrameLibWebRTC : public rtc::RefCountedObject<webrtc::VideoFrameBuffer> {
 public: