From 483d3533b710ee84c6fe782e6fa4e941d35ab424 Mon Sep 17 00:00:00 2001 From: will wade Date: Wed, 9 Sep 2026 07:39:30 +0100 Subject: [PATCH 1/3] feat(elevenlabs): tts_speak_ssml translates SSML into the dialect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ElevenLabs parses no SSML, so SSML input (the tts_speak_ssml path and SpeechMarkdown that expands to -wrapped markup) was stripped to plain text — every break, whisper and style lost. Now it round-trips through SpeechMarkdown into the model-matched dialect: breaks become pause tiers, prosody becomes tempo tags, amazon:effect whispered becomes [whispers], mstts:express-as styles pass through as audio tags (speechmarkdown-rust 0.4.16). Parse failures fall back to stripping. Verified end-to-end with the offline mock server: the request body for SSML input on the default eleven_v3 model contains the audio-tag dialect and no raw XML. --- Cargo.toml | 2 +- src/cloud_engine.rs | 55 +++++++++++++++++- tests/elevenlabs_timestamps_fallback.rs | 76 ++++++++++++++++++++++--- 3 files changed, 122 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f1bc17f..f8555b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,7 +97,7 @@ sherpa-onnx-models = { version = "0.2", optional = true } # Pin sys to match the main crate (cargo update can drift them apart). sherpa-onnx-sys = { version = "=1.13.5", optional = true } base64 = { version = "0.22", optional = true } -speechmarkdown-rust = { version = "0.4.14", optional = true } +speechmarkdown-rust = { version = "0.4.16", optional = true } anyhow = "1" tungstenite = { version = "0.29.0", features = ["rustls-tls-webpki-roots"], optional = true } uuid = { version = "1.23.2", features = ["v4"], optional = true } diff --git a/src/cloud_engine.rs b/src/cloud_engine.rs index 25726ee..a8d81fc 100644 --- a/src/cloud_engine.rs +++ b/src/cloud_engine.rs @@ -1431,6 +1431,18 @@ fn elevenlabs_smd_platform<'a>(provider: &'a str, model: Option<&str>) -> &'a st } } +/// Translate W3C (or Alexa/Azure-flavoured) SSML into an ElevenLabs +/// dialect prompt: SSML → SpeechMarkdown → the model-matched dialect. +/// Returns `None` when the input does not parse; callers then fall back +/// to plain-text stripping. +#[cfg(feature = "speechmarkdown")] +fn elevenlabs_ssml_to_dialect(ssml: &str, smd_platform: &str) -> Option { + use speechmarkdown_rust::{Platform, SpeechMarkdownParser}; + let platform = Platform::from_platform_str(smd_platform)?; + let smd = SpeechMarkdownParser::to_smd(ssml).ok()?; + SpeechMarkdownParser::to_ssml(&smd, platform).ok() +} + /// Parse ElevenLabs alignment payload into `(word, start_sec, end_sec)` tuples. /// /// ElevenLabs returns per-character timing in `alignment`: @@ -1893,7 +1905,20 @@ impl TtsEngine for CloudEngine { } _ => { google_ssml_override = None; - text = crate::engine::strip_ssml_to_text(&original_text); + // ElevenLabs parses no SSML: translate into the + // model-matched dialect (breaks, whisper, styles + // survive) rather than stripping to plain text. + #[cfg(feature = "speechmarkdown")] + if self.config.provider_id == "elevenlabs" { + text = elevenlabs_ssml_to_dialect(&original_text, smd_platform) + .unwrap_or_else(|| crate::engine::strip_ssml_to_text(&original_text)); + } else { + text = crate::engine::strip_ssml_to_text(&original_text); + } + #[cfg(not(feature = "speechmarkdown"))] + { + text = crate::engine::strip_ssml_to_text(&original_text); + } } } } else { @@ -5082,6 +5107,34 @@ mod tests { assert_eq!(cfg.model_default.as_deref(), Some("eleven_v3")); } + #[test] + fn test_elevenlabs_ssml_translates_to_dialects() { + // W3C/Alexa/Azure-flavoured SSML → SpeechMarkdown → the + // model-matched ElevenLabs dialect (used by the tts_speak_ssml + // path instead of stripping). + let alexa = r#"Hello world"#; + assert_eq!( + elevenlabs_ssml_to_dialect(alexa, "elevenlabs").unwrap(), + "Hello world" + ); + let v3 = r#"secret"#; + assert_eq!( + elevenlabs_ssml_to_dialect(v3, "elevenlabs-v3").unwrap(), + "[whispers] secret" + ); + let azure = r#"hi"#; + assert_eq!( + elevenlabs_ssml_to_dialect(azure, "elevenlabs-v3").unwrap(), + "[cheerful] hi" + ); + // Plain text parses as bare SpeechMarkdown and round-trips + // unchanged (no translation needed). + assert_eq!( + elevenlabs_ssml_to_dialect("no ssml here", "elevenlabs").unwrap(), + "no ssml here" + ); + } + #[test] fn test_elevenlabs_dialect_follows_model() { // The production predicate used by speak(): eleven_v3* → audio-tag diff --git a/tests/elevenlabs_timestamps_fallback.rs b/tests/elevenlabs_timestamps_fallback.rs index f76942e..d355c76 100644 --- a/tests/elevenlabs_timestamps_fallback.rs +++ b/tests/elevenlabs_timestamps_fallback.rs @@ -23,7 +23,12 @@ fn find_subsequence(haystack: &[u8], needle: &[u8]) -> Option { /// Minimal HTTP/1.1 responder: drains exactly one request (headers plus /// Content-Length body) and writes `status` with `body`. Returns the /// request line so tests can assert on the path. -fn respond(stream: &mut TcpStream, status: &str, content_type: &str, body: &[u8]) -> String { +fn respond( + stream: &mut TcpStream, + status: &str, + content_type: &str, + body: &[u8], +) -> (String, String) { let mut buf = vec![0u8; 16_384]; let mut read = 0usize; let header_end = loop { @@ -56,7 +61,10 @@ fn respond(stream: &mut TcpStream, status: &str, content_type: &str, body: &[u8] .expect("write headers"); stream.write_all(body).expect("write body"); stream.flush().expect("flush"); - headers.lines().next().unwrap_or_default().to_string() + let request_line = headers.lines().next().unwrap_or_default().to_string(); + let req_body = + String::from_utf8_lossy(&buf[header_end + 4..header_end + 4 + content_length]).to_string(); + (request_line, req_body) } #[test] @@ -67,15 +75,18 @@ fn elevenlabs_timestamps_rejection_degrades_to_estimated_boundaries() { let mut paths = Vec::new(); // First attempt: the /with-timestamps variant is rejected. let (mut stream, _) = listener.accept().expect("accept 1"); - paths.push(respond( - &mut stream, - "404 Not Found", - "application/json", - b"{\"detail\":{\"message\":\"not found\"}}", - )); + paths.push( + respond( + &mut stream, + "404 Not Found", + "application/json", + b"{\"detail\":{\"message\":\"not found\"}}", + ) + .0, + ); // Retry: plain synthesis endpoint with an MP3 body. let (mut stream, _) = listener.accept().expect("accept 2"); - paths.push(respond(&mut stream, "200 OK", "audio/mpeg", SILENCE_MP3)); + paths.push(respond(&mut stream, "200 OK", "audio/mpeg", SILENCE_MP3).0); paths }); @@ -119,3 +130,50 @@ fn elevenlabs_timestamps_rejection_degrades_to_estimated_boundaries() { ); assert_eq!(words, ["Hello", "boundary", "fallback"]); } + +#[test] +fn elevenlabs_ssml_input_is_translated_not_stripped() { + // The tts_speak_ssml path on ElevenLabs: W3C SSML must arrive at the + // API as the model-matched dialect (default model eleven_v3 → audio + // tags), not stripped to plain text and not as raw XML. + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().expect("local addr"); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("accept"); + respond(&mut stream, "200 OK", "audio/mpeg", SILENCE_MP3).1 + }); + + let creds = format!(r#"{{"apiKey":"test-key","synthUrl":"http://{addr}"}}"#); + let engine = create_engine("elevenlabs", &creds).expect("elevenlabs engine"); + let mut total = 0usize; + let mut on_audio = |chunk: &[u8]| { + total += chunk.len(); + }; + engine + .speak( + "Hello world", + None, + 1.0, + 1.0, + 1.0, + Some(&mut on_audio), + None, + None, + ) + .expect("speak with SSML input"); + + let body = server.join().expect("server thread"); + assert!( + body.contains("[long pause]"), + "break must become a v3 pause tag; body: {body}" + ); + assert!( + body.contains("[drawn out] world"), + "slow prosody must become a tempo tag; body: {body}" + ); + assert!( + !body.contains(" 0); +} From d3c952840567d98495856c2c35cae30894513b59 Mon Sep 17 00:00:00 2001 From: will wade Date: Wed, 9 Sep 2026 07:51:16 +0100 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20review=20round=20=E2=80=94=20degrade?= =?UTF-8?q?d=20boundaries=20from=20plain=20text,=20docs,=20fallback=20test?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review caught an estimated-boundary regression: with SSML input on the default eleven_v3 model (which lacks /with-timestamps), the estimator was tokenizing the translated dialect — boundary words like "[long" and "pause]". Boundary search text for translated ElevenLabs SSML is now the plain spoken text, restoring pre-translation semantics; pinned by a degraded-path integration test asserting exactly the plain words. Also: malformed-SSML → strip fallback and -dropped behavior pinned in unit tests; README FFI table corrected (ElevenLabs translates SSML via SpeechMarkdown rather than bypassing it); cbindgen header + vendored Swift copy synced. --- README.md | 2 +- src/cloud_engine.rs | 31 ++++++++++++++-- tests/elevenlabs_timestamps_fallback.rs | 49 +++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index eee3336..1091f6c 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ All functions are `extern "C"`, `#[no_mangle]`: | `tts_create(engine_id, credentials_json)` | Create engine, returns opaque `tts_ctx*` | | `tts_destroy(ctx)` | Free engine context | | `tts_speak(ctx, text)` | Speak (returns 0/-1) | -| `tts_speak_ssml(ctx, ssml)` | Speak pre-built SSML, bypassing SpeechMarkdown + rate/pitch/volume wrapping | +| `tts_speak_ssml(ctx, ssml)` | Speak pre-built SSML, bypassing rate/pitch/volume wrapping. Engines that parse SSML get it directly; ElevenLabs gets it translated via SpeechMarkdown into its model-matched dialect (it parses no SSML) | | `tts_speak_sync(ctx, text)` | Speak (blocking) | | `tts_stop(ctx)` | Stop speech | | `tts_pause(ctx)` | Pause in-progress speech | diff --git a/src/cloud_engine.rs b/src/cloud_engine.rs index a8d81fc..36d81b7 100644 --- a/src/cloud_engine.rs +++ b/src/cloud_engine.rs @@ -1884,7 +1884,9 @@ impl TtsEngine for CloudEngine { match self.config.provider_id.as_str() { "azure" | "edge" => { google_ssml_override = None; - text = original_text; + // Clone: original_text is still needed for boundary + // text mapping below. + text = original_text.clone(); } "google" => { let (v, inner) = crate::engine::unwrap_voice_tag(&original_text); @@ -1923,14 +1925,26 @@ impl TtsEngine for CloudEngine { } } else { google_ssml_override = None; - text = original_text; + // Clone: original_text stays available for boundary mapping. + text = original_text.clone(); } // Boundary word search target (see `user_text` above): plain or // dialect-reformatted input maps against what the caller passed; // SSML paths keep searching the processed string (previously - // existing behavior). - let boundary_search_text: &str = if is_ssml { text.as_str() } else { user_text }; + // existing behavior). Exception: ElevenLabs SSML input was + // translated into prompt markup — the dialect text (and its tag + // fragments) must not leak into boundary words or offset + // mapping; search the plain spoken text instead. + let plain_spoken; + let boundary_search_text: &str = if is_ssml && self.config.provider_id == "elevenlabs" { + plain_spoken = crate::engine::strip_ssml_to_text(&original_text); + plain_spoken.as_str() + } else if is_ssml { + text.as_str() + } else { + user_text + }; // WebSocket approach: Azure when word boundaries are requested, or // Edge always (Edge is WS-only — it has no REST synth endpoint). @@ -5133,6 +5147,15 @@ mod tests { elevenlabs_ssml_to_dialect("no ssml here", "elevenlabs").unwrap(), "no ssml here" ); + // Malformed SSML fails to parse → None → the caller strips. + assert!(elevenlabs_ssml_to_dialect("a & b", "elevenlabs").is_none()); + // has no ElevenLabs equivalent (parity with the old + // strip path): the modifier is dropped, the text survives. + let voiced = r#"hi"#; + assert_eq!( + elevenlabs_ssml_to_dialect(voiced, "elevenlabs-v3").unwrap(), + "hi" + ); } #[test] diff --git a/tests/elevenlabs_timestamps_fallback.rs b/tests/elevenlabs_timestamps_fallback.rs index d355c76..444e081 100644 --- a/tests/elevenlabs_timestamps_fallback.rs +++ b/tests/elevenlabs_timestamps_fallback.rs @@ -177,3 +177,52 @@ fn elevenlabs_ssml_input_is_translated_not_stripped() { ); assert!(total > 0); } + +#[test] +fn elevenlabs_ssml_input_degraded_boundaries_are_plain_words() { + // SSML input + boundaries on the default eleven_v3 model: the + // /with-timestamps variant is rejected, so boundaries come from the + // estimator — and must be the plain spoken words, never dialect tag + // fragments like "[long" or "pause]". + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().expect("local addr"); + let server = std::thread::spawn(move || { + let mut paths = Vec::new(); + let (mut stream, _) = listener.accept().expect("accept 1"); + paths.push( + respond( + &mut stream, + "404 Not Found", + "application/json", + b"{\"detail\":{\"message\":\"not found\"}}", + ) + .0, + ); + let (mut stream, _) = listener.accept().expect("accept 2"); + paths.push(respond(&mut stream, "200 OK", "audio/mpeg", SILENCE_MP3).0); + paths + }); + + let creds = format!(r#"{{"apiKey":"test-key","synthUrl":"http://{addr}"}}"#); + let engine = create_engine("elevenlabs", &creds).expect("elevenlabs engine"); + let mut words: Vec = Vec::new(); + let mut on_boundary = |word: &str, _s: f32, _e: f32, _off: i32, _len: i32, _est: bool| { + words.push(word.to_string()); + }; + engine + .speak( + "Hello world", + None, + 1.0, + 1.0, + 1.0, + None, + Some(&mut on_boundary), + None, + ) + .expect("degraded speak with SSML input"); + + let paths = server.join().expect("server thread"); + assert!(paths[0].contains("/with-timestamps")); + assert_eq!(words, ["Hello", "world"]); +} From 852794b19490b5fb21c76c58a4786d0e9be8e479 Mon Sep 17 00:00:00 2001 From: will wade Date: Wed, 9 Sep 2026 07:51:53 +0100 Subject: [PATCH 3/3] docs: tts_speak_ssml contract covers the ElevenLabs translation; headers synced --- .../swift/Sources/CRustTtsWrapper/include/tts_wrapper.h | 8 +++++--- include/tts_wrapper.h | 8 +++++--- src/lib.rs | 8 +++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/bindings/swift/Sources/CRustTtsWrapper/include/tts_wrapper.h b/bindings/swift/Sources/CRustTtsWrapper/include/tts_wrapper.h index b737f40..930c3df 100644 --- a/bindings/swift/Sources/CRustTtsWrapper/include/tts_wrapper.h +++ b/bindings/swift/Sources/CRustTtsWrapper/include/tts_wrapper.h @@ -137,9 +137,11 @@ int32_t tts_speak(struct tts_ctx *ctx, const char *text); /** * Speak pre-built SSML using the engine in `ctx`. * - * The SSML is passed directly to the engine without SpeechMarkdown - * conversion or rate/pitch/volume wrapping. Callers are responsible - * for embedding all prosody in the SSML. + * The SSML bypasses rate/pitch/volume wrapping; callers are + * responsible for embedding all prosody in the SSML. Engines that + * parse SSML receive it directly; engines that do not (ElevenLabs) + * get it translated via SpeechMarkdown into their model-matched + * prompt dialect. * * Returns 0 on success, -1 on failure. * diff --git a/include/tts_wrapper.h b/include/tts_wrapper.h index b737f40..930c3df 100644 --- a/include/tts_wrapper.h +++ b/include/tts_wrapper.h @@ -137,9 +137,11 @@ int32_t tts_speak(struct tts_ctx *ctx, const char *text); /** * Speak pre-built SSML using the engine in `ctx`. * - * The SSML is passed directly to the engine without SpeechMarkdown - * conversion or rate/pitch/volume wrapping. Callers are responsible - * for embedding all prosody in the SSML. + * The SSML bypasses rate/pitch/volume wrapping; callers are + * responsible for embedding all prosody in the SSML. Engines that + * parse SSML receive it directly; engines that do not (ElevenLabs) + * get it translated via SpeechMarkdown into their model-matched + * prompt dialect. * * Returns 0 on success, -1 on failure. * diff --git a/src/lib.rs b/src/lib.rs index 70f29ba..a9ddeb3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -338,9 +338,11 @@ pub extern "C" fn tts_speak(ctx: *mut tts_ctx, text: *const c_char) -> i32 { /// Speak pre-built SSML using the engine in `ctx`. /// -/// The SSML is passed directly to the engine without SpeechMarkdown -/// conversion or rate/pitch/volume wrapping. Callers are responsible -/// for embedding all prosody in the SSML. +/// The SSML bypasses rate/pitch/volume wrapping; callers are +/// responsible for embedding all prosody in the SSML. Engines that +/// parse SSML receive it directly; engines that do not (ElevenLabs) +/// get it translated via SpeechMarkdown into their model-matched +/// prompt dialect. /// /// Returns 0 on success, -1 on failure. ///