ElevenLabs: translate SSML input into the model-matched dialect

Cargo.toml

@@ -97,7 +97,7 @@ sherpa-onnx-models = { version = "0.2", optional = true }
 # Pin sys to match the main crate (cargo update can drift them apart).
 sherpa-onnx-sys = { version = "=1.13.5", optional = true }
 base64 = { version = "0.22", optional = true }
-speechmarkdown-rust = { version = "0.4.14", optional = true }
+speechmarkdown-rust = { version = "0.4.16", optional = true }
 anyhow = "1"
 tungstenite = { version = "0.29.0", features = ["rustls-tls-webpki-roots"], optional = true }
 uuid = { version = "1.23.2", features = ["v4"], optional = true }

README.md

@@ -169,7 +169,7 @@ All functions are `extern "C"`, `#[no_mangle]`:
 | `tts_create(engine_id, credentials_json)` | Create engine, returns opaque `tts_ctx*` |
 | `tts_destroy(ctx)` | Free engine context |
 | `tts_speak(ctx, text)` | Speak (returns 0/-1) |
-| `tts_speak_ssml(ctx, ssml)` | Speak pre-built SSML, bypassing SpeechMarkdown + rate/pitch/volume wrapping |
+| `tts_speak_ssml(ctx, ssml)` | Speak pre-built SSML, bypassing rate/pitch/volume wrapping. Engines that parse SSML get it directly; ElevenLabs gets it translated via SpeechMarkdown into its model-matched dialect (it parses no SSML) |
 | `tts_speak_sync(ctx, text)` | Speak (blocking) |
 | `tts_stop(ctx)` | Stop speech |
 | `tts_pause(ctx)` | Pause in-progress speech |

bindings/swift/Sources/CRustTtsWrapper/include/tts_wrapper.h

@@ -137,9 +137,11 @@ int32_t tts_speak(struct tts_ctx *ctx, const char *text);
 /**
  * Speak pre-built SSML using the engine in `ctx`.
  *
- * The SSML is passed directly to the engine without SpeechMarkdown
- * conversion or rate/pitch/volume wrapping. Callers are responsible
- * for embedding all prosody in the SSML.
+ * The SSML bypasses rate/pitch/volume wrapping; callers are
+ * responsible for embedding all prosody in the SSML. Engines that
+ * parse SSML receive it directly; engines that do not (ElevenLabs)
+ * get it translated via SpeechMarkdown into their model-matched
+ * prompt dialect.
  *
  * Returns 0 on success, -1 on failure.
  *

include/tts_wrapper.h

@@ -137,9 +137,11 @@ int32_t tts_speak(struct tts_ctx *ctx, const char *text);
 /**
  * Speak pre-built SSML using the engine in `ctx`.
  *
- * The SSML is passed directly to the engine without SpeechMarkdown
- * conversion or rate/pitch/volume wrapping. Callers are responsible
- * for embedding all prosody in the SSML.
+ * The SSML bypasses rate/pitch/volume wrapping; callers are
+ * responsible for embedding all prosody in the SSML. Engines that
+ * parse SSML receive it directly; engines that do not (ElevenLabs)
+ * get it translated via SpeechMarkdown into their model-matched
+ * prompt dialect.
  *
  * Returns 0 on success, -1 on failure.
  *

src/cloud_engine.rs

@@ -1431,6 +1431,18 @@ fn elevenlabs_smd_platform<'a>(provider: &'a str, model: Option<&str>) -> &'a st
     }
 }
 
+/// Translate W3C (or Alexa/Azure-flavoured) SSML into an ElevenLabs
+/// dialect prompt: SSML → SpeechMarkdown → the model-matched dialect.
+/// Returns `None` when the input does not parse; callers then fall back
+/// to plain-text stripping.
+#[cfg(feature = "speechmarkdown")]
+fn elevenlabs_ssml_to_dialect(ssml: &str, smd_platform: &str) -> Option<String> {
+    use speechmarkdown_rust::{Platform, SpeechMarkdownParser};
+    let platform = Platform::from_platform_str(smd_platform)?;
+    let smd = SpeechMarkdownParser::to_smd(ssml).ok()?;
+    SpeechMarkdownParser::to_ssml(&smd, platform).ok()
+}
+
 /// Parse ElevenLabs alignment payload into `(word, start_sec, end_sec)` tuples.
 ///
 /// ElevenLabs returns per-character timing in `alignment`:
@@ -1872,7 +1884,9 @@ impl TtsEngine for CloudEngine {
             match self.config.provider_id.as_str() {
                 "azure" | "edge" => {
                     google_ssml_override = None;
-                    text = original_text;
+                    // Clone: original_text is still needed for boundary
+                    // text mapping below.
+                    text = original_text.clone();
                 }
                 "google" => {
                     let (v, inner) = crate::engine::unwrap_voice_tag(&original_text);
@@ -1893,19 +1907,44 @@ impl TtsEngine for CloudEngine {
                 }
                 _ => {
                     google_ssml_override = None;
-                    text = crate::engine::strip_ssml_to_text(&original_text);
+                    // ElevenLabs parses no SSML: translate into the
+                    // model-matched dialect (breaks, whisper, styles
+                    // survive) rather than stripping to plain text.
+                    #[cfg(feature = "speechmarkdown")]
+                    if self.config.provider_id == "elevenlabs" {
+                        text = elevenlabs_ssml_to_dialect(&original_text, smd_platform)
+                            .unwrap_or_else(|| crate::engine::strip_ssml_to_text(&original_text));
+                    } else {
+                        text = crate::engine::strip_ssml_to_text(&original_text);
+                    }
+                    #[cfg(not(feature = "speechmarkdown"))]
+                    {
+                        text = crate::engine::strip_ssml_to_text(&original_text);
+                    }
                 }
             }
         } else {
             google_ssml_override = None;
-            text = original_text;
+            // Clone: original_text stays available for boundary mapping.
+            text = original_text.clone();
         }
 
         // Boundary word search target (see `user_text` above): plain or
         // dialect-reformatted input maps against what the caller passed;
         // SSML paths keep searching the processed string (previously
-        // existing behavior).
-        let boundary_search_text: &str = if is_ssml { text.as_str() } else { user_text };
+        // existing behavior). Exception: ElevenLabs SSML input was
+        // translated into prompt markup — the dialect text (and its tag
+        // fragments) must not leak into boundary words or offset
+        // mapping; search the plain spoken text instead.
+        let plain_spoken;
+        let boundary_search_text: &str = if is_ssml && self.config.provider_id == "elevenlabs" {
+            plain_spoken = crate::engine::strip_ssml_to_text(&original_text);
+            plain_spoken.as_str()
+        } else if is_ssml {
+            text.as_str()
+        } else {
+            user_text
+        };
 
         // WebSocket approach: Azure when word boundaries are requested, or
         // Edge always (Edge is WS-only — it has no REST synth endpoint).
@@ -5082,6 +5121,43 @@ mod tests {
         assert_eq!(cfg.model_default.as_deref(), Some("eleven_v3"));
     }
 
+    #[test]
+    fn test_elevenlabs_ssml_translates_to_dialects() {
+        // W3C/Alexa/Azure-flavoured SSML → SpeechMarkdown → the
+        // model-matched ElevenLabs dialect (used by the tts_speak_ssml
+        // path instead of stripping).
+        let alexa = r#"<speak>Hello <break time="2s"/> world</speak>"#;
+        assert_eq!(
+            elevenlabs_ssml_to_dialect(alexa, "elevenlabs").unwrap(),
+            "Hello <break time=\"2s\"/> world"
+        );
+        let v3 = r#"<speak><amazon:effect name="whispered">secret</amazon:effect></speak>"#;
+        assert_eq!(
+            elevenlabs_ssml_to_dialect(v3, "elevenlabs-v3").unwrap(),
+            "[whispers] secret"
+        );
+        let azure = r#"<speak><mstts:express-as style="cheerful">hi</mstts:express-as></speak>"#;
+        assert_eq!(
+            elevenlabs_ssml_to_dialect(azure, "elevenlabs-v3").unwrap(),
+            "[cheerful] hi"
+        );
+        // Plain text parses as bare SpeechMarkdown and round-trips
+        // unchanged (no translation needed).
+        assert_eq!(
+            elevenlabs_ssml_to_dialect("no ssml here", "elevenlabs").unwrap(),
+            "no ssml here"
+        );
+        // Malformed SSML fails to parse → None → the caller strips.
+        assert!(elevenlabs_ssml_to_dialect("<speak>a & b</speak>", "elevenlabs").is_none());
+        // <voice> has no ElevenLabs equivalent (parity with the old
+        // strip path): the modifier is dropped, the text survives.
+        let voiced = r#"<speak><voice name="Aria">hi</voice></speak>"#;
+        assert_eq!(
+            elevenlabs_ssml_to_dialect(voiced, "elevenlabs-v3").unwrap(),
+            "hi"
+        );
+    }
+
     #[test]
     fn test_elevenlabs_dialect_follows_model() {
         // The production predicate used by speak(): eleven_v3* → audio-tag

src/lib.rs

@@ -338,9 +338,11 @@ pub extern "C" fn tts_speak(ctx: *mut tts_ctx, text: *const c_char) -> i32 {
 
 /// Speak pre-built SSML using the engine in `ctx`.
 ///
-/// The SSML is passed directly to the engine without SpeechMarkdown
-/// conversion or rate/pitch/volume wrapping. Callers are responsible
-/// for embedding all prosody in the SSML.
+/// The SSML bypasses rate/pitch/volume wrapping; callers are
+/// responsible for embedding all prosody in the SSML. Engines that
+/// parse SSML receive it directly; engines that do not (ElevenLabs)
+/// get it translated via SpeechMarkdown into their model-matched
+/// prompt dialect.
 ///
 /// Returns 0 on success, -1 on failure.
 ///

tests/elevenlabs_timestamps_fallback.rs

@@ -23,7 +23,12 @@ fn find_subsequence(haystack: &[u8], needle: &[u8]) -> Option<usize> {
 /// Minimal HTTP/1.1 responder: drains exactly one request (headers plus
 /// Content-Length body) and writes `status` with `body`. Returns the
 /// request line so tests can assert on the path.
-fn respond(stream: &mut TcpStream, status: &str, content_type: &str, body: &[u8]) -> String {
+fn respond(
+    stream: &mut TcpStream,
+    status: &str,
+    content_type: &str,
+    body: &[u8],
+) -> (String, String) {
     let mut buf = vec![0u8; 16_384];
     let mut read = 0usize;
     let header_end = loop {
@@ -56,7 +61,10 @@ fn respond(stream: &mut TcpStream, status: &str, content_type: &str, body: &[u8]
         .expect("write headers");
     stream.write_all(body).expect("write body");
     stream.flush().expect("flush");
-    headers.lines().next().unwrap_or_default().to_string()
+    let request_line = headers.lines().next().unwrap_or_default().to_string();
+    let req_body =
+        String::from_utf8_lossy(&buf[header_end + 4..header_end + 4 + content_length]).to_string();
+    (request_line, req_body)
 }
 
 #[test]
@@ -67,15 +75,18 @@ fn elevenlabs_timestamps_rejection_degrades_to_estimated_boundaries() {
         let mut paths = Vec::new();
         // First attempt: the /with-timestamps variant is rejected.
         let (mut stream, _) = listener.accept().expect("accept 1");
-        paths.push(respond(
-            &mut stream,
-            "404 Not Found",
-            "application/json",
-            b"{\"detail\":{\"message\":\"not found\"}}",
-        ));
+        paths.push(
+            respond(
+                &mut stream,
+                "404 Not Found",
+                "application/json",
+                b"{\"detail\":{\"message\":\"not found\"}}",
+            )
+            .0,
+        );
         // Retry: plain synthesis endpoint with an MP3 body.
         let (mut stream, _) = listener.accept().expect("accept 2");
-        paths.push(respond(&mut stream, "200 OK", "audio/mpeg", SILENCE_MP3));
+        paths.push(respond(&mut stream, "200 OK", "audio/mpeg", SILENCE_MP3).0);
         paths
     });
 
@@ -119,3 +130,99 @@ fn elevenlabs_timestamps_rejection_degrades_to_estimated_boundaries() {
     );
     assert_eq!(words, ["Hello", "boundary", "fallback"]);
 }
+
+#[test]
+fn elevenlabs_ssml_input_is_translated_not_stripped() {
+    // The tts_speak_ssml path on ElevenLabs: W3C SSML must arrive at the
+    // API as the model-matched dialect (default model eleven_v3 → audio
+    // tags), not stripped to plain text and not as raw XML.
+    let listener = TcpListener::bind("127.0.0.1:0").expect("bind");
+    let addr = listener.local_addr().expect("local addr");
+    let server = std::thread::spawn(move || {
+        let (mut stream, _) = listener.accept().expect("accept");
+        respond(&mut stream, "200 OK", "audio/mpeg", SILENCE_MP3).1
+    });
+
+    let creds = format!(r#"{{"apiKey":"test-key","synthUrl":"http://{addr}"}}"#);
+    let engine = create_engine("elevenlabs", &creds).expect("elevenlabs engine");
+    let mut total = 0usize;
+    let mut on_audio = |chunk: &[u8]| {
+        total += chunk.len();
+    };
+    engine
+        .speak(
+            "<speak>Hello <break time=\"2s\"/> <prosody rate=\"slow\">world</prosody></speak>",
+            None,
+            1.0,
+            1.0,
+            1.0,
+            Some(&mut on_audio),
+            None,
+            None,
+        )
+        .expect("speak with SSML input");
+
+    let body = server.join().expect("server thread");
+    assert!(
+        body.contains("[long pause]"),
+        "break must become a v3 pause tag; body: {body}"
+    );
+    assert!(
+        body.contains("[drawn out] world"),
+        "slow prosody must become a tempo tag; body: {body}"
+    );
+    assert!(
+        !body.contains("<break"),
+        "raw SSML must not reach the API; body: {body}"
+    );
+    assert!(total > 0);
+}
+
+#[test]
+fn elevenlabs_ssml_input_degraded_boundaries_are_plain_words() {
+    // SSML input + boundaries on the default eleven_v3 model: the
+    // /with-timestamps variant is rejected, so boundaries come from the
+    // estimator — and must be the plain spoken words, never dialect tag
+    // fragments like "[long" or "pause]".
+    let listener = TcpListener::bind("127.0.0.1:0").expect("bind");
+    let addr = listener.local_addr().expect("local addr");
+    let server = std::thread::spawn(move || {
+        let mut paths = Vec::new();
+        let (mut stream, _) = listener.accept().expect("accept 1");
+        paths.push(
+            respond(
+                &mut stream,
+                "404 Not Found",
+                "application/json",
+                b"{\"detail\":{\"message\":\"not found\"}}",
+            )
+            .0,
+        );
+        let (mut stream, _) = listener.accept().expect("accept 2");
+        paths.push(respond(&mut stream, "200 OK", "audio/mpeg", SILENCE_MP3).0);
+        paths
+    });
+
+    let creds = format!(r#"{{"apiKey":"test-key","synthUrl":"http://{addr}"}}"#);
+    let engine = create_engine("elevenlabs", &creds).expect("elevenlabs engine");
+    let mut words: Vec<String> = Vec::new();
+    let mut on_boundary = |word: &str, _s: f32, _e: f32, _off: i32, _len: i32, _est: bool| {
+        words.push(word.to_string());
+    };
+    engine
+        .speak(
+            "<speak>Hello <break time=\"2s\"/> world</speak>",
+            None,
+            1.0,
+            1.0,
+            1.0,
+            None,
+            Some(&mut on_boundary),
+            None,
+        )
+        .expect("degraded speak with SSML input");
+
+    let paths = server.join().expect("server thread");
+    assert!(paths[0].contains("/with-timestamps"));
+    assert_eq!(words, ["Hello", "world"]);
+}