Rialto with WebKit audio sink https://bugs.webkit.org/show_bug.cgi?id=278911

Reviewed by Philippe Normand.

Rialto supports single instance of webaudio pcm sing so it is not possible to create more than one WebAudio context. Use
WebKit audio sink instead, that mixes all webaudio contextes into single stream and pass it to rialto webaudio sink.

With webkit 2.38 webaudio src element produce 'non-interleaved' audio format that is not fully supported by some
platform sinks (audio output is breaking). WebKit audio sink (mixer pipeline) also doesn't seem to handle that
correctly. Mixer pipeline fails to negotiate caps and inter audio sink reqires the latest gst version to work with such
stream.

Patch by Andrzej Surdej <Andrzej_Surdej@comcast.com>.

* Source/WebCore/platform/audio/gstreamer/AudioDestinationGStreamer.cpp:
(WebCore::AudioDestinationGStreamer::AudioDestinationGStreamer):
* Source/WebCore/platform/gstreamer/GStreamerQuirkRialto.cpp:
(WebCore::GStreamerQuirkRialto::createWebAudioSink):

Canonical link: https://commits.webkit.org/283137@main

Author: calvaris

Source/WebCore/platform/audio/gstreamer/AudioDestinationGStreamer.cpp

@@ -157,10 +157,24 @@ AudioDestinationGStreamer::AudioDestinationGStreamer(AudioIOCallback& callback,
 
     gst_bin_add_many(GST_BIN_CAST(m_pipeline.get()), m_src.get(), audioConvert, audioResample, queue, audioSink.get(), nullptr);
 
-    // Link src pads from webkitAudioSrc to audioConvert ! audioResample ! queue ! autoaudiosink.
+    // Link src pads from webkitAudioSrc to audioConvert ! audioResample ! [capsfilter !] queue ! autoaudiosink.
     gst_element_link_pads_full(m_src.get(), "src", audioConvert, "sink", GST_PAD_LINK_CHECK_NOTHING);
     gst_element_link_pads_full(audioConvert, "src", audioResample, "sink", GST_PAD_LINK_CHECK_NOTHING);
-    gst_element_link_pads_full(audioResample, "src", queue, "sink", GST_PAD_LINK_CHECK_NOTHING);
+
+    if (!webkitGstCheckVersion(1, 20, 4)) {
+        // Force audio conversion to 'interleaved' format (by audioconvert element).
+        // 1) Some platform sinks don't support non-interleaved audio without special caps (rialtowebaudiosink).
+        // 2) Interaudio sink/src doesn't fully support non-interleaved audio (webkit audio sink)
+        // 3) audiomixer doesn't support non-interleaved audio in output pipeline (webkit audio sink)
+        GstElement* capsFilter = makeGStreamerElement("capsfilter", nullptr);
+        GRefPtr<GstCaps> caps = adoptGRef(gst_caps_new_simple("audio/x-raw", "layout", G_TYPE_STRING, "interleaved", nullptr));
+        g_object_set(capsFilter, "caps", caps.get(), nullptr);
+        gst_bin_add(GST_BIN_CAST(m_pipeline.get()), capsFilter);
+        gst_element_link_pads_full(audioResample, "src", capsFilter, "sink", GST_PAD_LINK_CHECK_NOTHING);
+        gst_element_link_pads_full(capsFilter, "src", queue, "sink", GST_PAD_LINK_CHECK_NOTHING);
+    } else
+        gst_element_link_pads_full(audioResample, "src", queue, "sink", GST_PAD_LINK_CHECK_NOTHING);
+
     gst_element_link_pads_full(queue, "src", audioSink.get(), "sink", GST_PAD_LINK_CHECK_NOTHING);
 }
 

Source/WebCore/platform/gstreamer/GStreamerQuirkRialto.cpp

@@ -29,6 +29,7 @@
 #if USE(GSTREAMER)
 
 #include "GStreamerCommon.h"
+#include "WebKitAudioSinkGStreamer.h"
 #include <wtf/OptionSet.h>
 
 namespace WebCore {
@@ -95,6 +96,9 @@ GstElement* GStreamerQuirkRialto::createAudioSink()
 
 GstElement* GStreamerQuirkRialto::createWebAudioSink()
 {
+    if (GstElement* sink = webkitAudioSinkNew())
+        return sink;
+
     auto sink = makeGStreamerElement("rialtowebaudiosink", nullptr);
     RELEASE_ASSERT_WITH_MESSAGE(sink, "rialtowebaudiosink should be available in the system but it is not");
     return sink;