diff --git a/apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/tts.py b/apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/tts.py index 32434b9217e..ad32ec35de6 100644 --- a/apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/tts.py +++ b/apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/tts.py @@ -75,6 +75,59 @@ class AliyunBaiLianQwenTTSModelGeneralParams(BaseForm): ) +def _qwen_audio_speech_rate_field(): + return SliderField( + TooltipLabel(_("Speaking speed"), _("[0.5, 2], the default is 1, usually one decimal place is enough")), + required=True, + default_value=1, + _min=0.5, + _max=2, + _step=0.1, + precision=1, + ) + + +class AliyunBaiLianQwenAudioTTSNextParams(BaseForm): + """Parameters for Qwen-Audio audio generation; voice is described in the prompt.""" + + speech_rate = _qwen_audio_speech_rate_field() + + +class AliyunBaiLianQwenAudio31TTSModelParams(BaseForm): + voice = SingleSelect( + TooltipLabel(_("Timbre"), _("Please select a system voice supported by Qwen-TTS")), + required=True, + default_value="longanhuan_v3.1", + text_field="value", + value_field="value", + option_list=[ + {"label": "longanhuan_v3.1", "value": "longanhuan_v3.1"}, + {"label": "longanlingxin_v3.1", "value": "longanlingxin_v3.1"}, + {"label": "longanfengyue_v3.1", "value": "longanfengyue_v3.1"}, + {"label": "xunanchuan_v3.1", "value": "xunanchuan_v3.1"}, + ], + ) + + speech_rate = _qwen_audio_speech_rate_field() + + +class AliyunBaiLianQwenAudio30TTSModelParams(BaseForm): + voice = SingleSelect( + TooltipLabel(_("Timbre"), _("Please select a system voice supported by Qwen-TTS")), + required=True, + default_value="longanlingxin", + text_field="value", + value_field="value", + option_list=[ + {"label": "longanlingxin", "value": "longanlingxin"}, + {"label": "longanlufeng", "value": "longanlufeng"}, + {"label": "longanfengyue", "value": "longanfengyue"}, + ], + ) + + speech_rate = _qwen_audio_speech_rate_field() + + class AliyunBaiLianTTSModelCredential(BaseForm, BaseModelCredential): """ Credential class for the Aliyun BaiLian TTS (Text-to-Speech) model. @@ -153,6 +206,12 @@ def get_model_params_setting_form(self, model_name: str): :param model_name: Name of the model. :return: Parameter setting form. """ + if "qwen-audio" in model_name and "tts-next" in model_name: + return AliyunBaiLianQwenAudioTTSNextParams() + if "qwen-audio-3.0" in model_name: + return AliyunBaiLianQwenAudio30TTSModelParams() + if "qwen-audio-3.1" in model_name: + return AliyunBaiLianQwenAudio31TTSModelParams() if "qwen" in model_name and "tts" in model_name: return AliyunBaiLianQwenTTSModelGeneralParams() return AliyunBaiLianTTSModelGeneralParams() diff --git a/apps/models_provider/impl/aliyun_bai_lian_model_provider/model/tts.py b/apps/models_provider/impl/aliyun_bai_lian_model_provider/model/tts.py index fa97f3ee764..d5afec3dab7 100644 --- a/apps/models_provider/impl/aliyun_bai_lian_model_provider/model/tts.py +++ b/apps/models_provider/impl/aliyun_bai_lian_model_provider/model/tts.py @@ -1,4 +1,8 @@ +import base64 +import threading +from http import HTTPStatus from typing import Dict +from urllib.parse import urlsplit, urlunsplit import dashscope @@ -9,6 +13,36 @@ from models_provider.impl.base_tts import BaseTextToSpeech +class _QwenTtsRealtimeCallback: + def __init__(self): + self.audio_chunks = [] + self.error = None + self.finished = threading.Event() + + def on_open(self): + pass + + def on_event(self, event): + event_type = event.get("type") + if event_type == "response.audio.delta": + try: + self.audio_chunks.append(base64.b64decode(event["delta"])) + except (KeyError, ValueError) as e: + self.error = "Failed to decode realtime TTS audio: %s" % e + self.finished.set() + elif event_type == "error": + error = event.get("error") or {} + self.error = "code: %s, message: %s" % (error.get("code"), error.get("message")) + self.finished.set() + elif event_type == "session.finished": + self.finished.set() + + def on_close(self, close_status_code, close_msg): + if close_status_code not in (None, 1000, 1001) and self.error is None: + self.error = "WebSocket closed, status_code: %s, message: %s" % (close_status_code, close_msg) + self.finished.set() + + class AliyunBaiLianTextToSpeech(MaxKBBaseModel, BaseTextToSpeech): api_key: str base_url: str @@ -41,10 +75,89 @@ def new_instance(model_type, model_name, model_credential: Dict[str, object], ** ) def check_auth(self): - self.text_to_speech(_("Hello")) + if "qwen-audio" in self.model and "tts-next" in self.model: + self.text_to_speech("一位女性清晰地说：“你好，欢迎使用。”") + else: + self.text_to_speech(_("Hello")) + + def _get_qwen_realtime_url(self): + parsed_url = urlsplit(self.base_url) + scheme = "wss" if parsed_url.scheme == "https" else "ws" + return urlunsplit((scheme, parsed_url.netloc, "/api-ws/v1/realtime", "", "")) + + def _text_to_speech_qwen_realtime(self, text): + from dashscope.audio.qwen_tts_realtime import QwenTtsRealtime + + params = dict(self.params or {}) + voice = params.pop("voice", "Cherry") + sample_rate = int(params.pop("sample_rate", 24000)) + params.pop("audio_format", None) + params.pop("response_format", None) + + callback = _QwenTtsRealtimeCallback() + synthesizer = QwenTtsRealtime( + model=self.model, + callback=callback, + url=self._get_qwen_realtime_url(), + ) + try: + synthesizer.connect() + synthesizer.update_session( + voice=voice, + mode="commit", + audio_format="mp3", + sample_rate=sample_rate, + **params, + ) + synthesizer.append_text(text) + synthesizer.commit() + synthesizer.finish() + if not callback.finished.wait(timeout=30): + raise TimeoutError("Qwen realtime TTS timed out after 30 seconds") + if callback.error: + raise Exception("Qwen realtime TTS request failed, %s" % callback.error) + audio = b"".join(callback.audio_chunks) + if not audio: + raise Exception("Qwen realtime TTS returned no audio") + return audio + finally: + synthesizer.close() + + def _text_to_speech_qwen_audio(self, text): + import requests + + params = dict(self.params or {}) + if "speech_rate" in params and "rate" not in params: + params["rate"] = params.pop("speech_rate") + + if "tts-next" in self.model: + params.pop("voice", None) + payload_input = {"text_prompt": text, **params} + else: + payload_input = {"text": text, **params} + + response = requests.post( + f"{self.base_url.rstrip('/')}/services/audio/tts/SpeechSynthesizer", + headers={"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"}, + json={"model": self.model, "input": payload_input}, + timeout=300, + ) + if response.status_code != HTTPStatus.OK: + error = response.json() + raise Exception( + "Qwen Audio TTS request failed, status_code: %s, code: %s, message: %s" + % (response.status_code, error.get("code"), error.get("message")) + ) + + audio_url = ((response.json().get("output") or {}).get("audio") or {}).get("url") + if not audio_url: + raise Exception("Qwen Audio TTS response does not contain an audio URL") + audio = requests.get(audio_url, timeout=30) + audio.raise_for_status() + return audio.content def text_to_speech(self, text): - global audio + audio = None dashscope.api_key = self.api_key dashscope.base_http_api_url = self.base_url text = _remove_empty_lines(text) @@ -52,6 +165,10 @@ def text_to_speech(self, text): from dashscope.audio.tts import SpeechSynthesizer audio = SpeechSynthesizer.call(model=self.model, text=text, **self.params).get_audio_data() + elif "qwen" in self.model and "realtime" in self.model: + audio = self._text_to_speech_qwen_realtime(text) + elif "qwen-audio" in self.model and "tts" in self.model: + audio = self._text_to_speech_qwen_audio(text) elif "qwen" in self.model: import requests @@ -62,9 +179,15 @@ def text_to_speech(self, text): text=text, **self.params, ) + if response.status_code != HTTPStatus.OK: + raise Exception( + "Qwen TTS request failed, status_code: %s, code: %s, message: %s" + % (response.status_code, response.code, response.message) + ) # 这个的接口返回格式和上面两个不太一样，直接返回了一个url地址，下载后就是音频文件 audio_url = response.output.audio.url - res = requests.get(audio_url) + res = requests.get(audio_url, timeout=30) + res.raise_for_status() audio = res.content elif "MiniMax" in self.model: import requests @@ -72,7 +195,8 @@ def text_to_speech(self, text): api_url = f"{self.base_url}/services/aigc/multimodal-generation/generation" headers = {"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"} payload = {"model": self.model, "input": {"text": text, **self.params}} - response = requests.post(api_url, headers=headers, json=payload) + response = requests.post(api_url, headers=headers, json=payload, timeout=30) + response.raise_for_status() audio_hex = response.json().get("output", {}).get("data", {}).get("audio") if audio_hex: audio = bytes.fromhex(audio_hex) @@ -85,6 +209,6 @@ def text_to_speech(self, text): audio = synthesizer.call(text) if audio is None: raise Exception("Failed to generate audio") - if type(audio) == str: + if isinstance(audio, str): raise Exception(audio) return audio diff --git a/apps/models_provider/tests/test_aliyun_tts.py b/apps/models_provider/tests/test_aliyun_tts.py new file mode 100644 index 00000000000..b26ad27789f --- /dev/null +++ b/apps/models_provider/tests/test_aliyun_tts.py @@ -0,0 +1,242 @@ +import base64 +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from django.conf import settings + + +class AliyunBaiLianTtsTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + if not settings.configured: + settings.configure(USE_I18N=False) + + def test_qwen_tts_reports_dashscope_error_instead_of_missing_output(self): + from models_provider.impl.aliyun_bai_lian_model_provider.model.tts import AliyunBaiLianTextToSpeech + + model = AliyunBaiLianTextToSpeech( + api_key="invalid-key", + base_url="https://dashscope.aliyuncs.com/api/v1", + model="qwen3-tts-flash", + params={}, + ) + response = SimpleNamespace( + status_code=401, + code="InvalidApiKey", + message="Invalid API-key provided.", + output=None, + ) + + with patch("dashscope.MultiModalConversation.call", return_value=response): + with self.assertRaisesRegex( + Exception, + r"status_code: 401, code: InvalidApiKey, message: Invalid API-key provided\.", + ): + model.text_to_speech("Hello") + + def test_qwen_realtime_tts_returns_collected_audio(self): + from models_provider.impl.aliyun_bai_lian_model_provider.model.tts import AliyunBaiLianTextToSpeech + + model = AliyunBaiLianTextToSpeech( + api_key="test-key", + base_url="https://dashscope-intl.aliyuncs.com/api/v1", + model="qwen3-tts-flash-realtime", + params={"voice": "Cherry", "speech_rate": 1}, + ) + state = {} + + class FakeQwenTtsRealtime: + def __init__(self, model, callback, url): + state.update(model=model, callback=callback, url=url) + + def connect(self): + state["connected"] = True + + def update_session(self, **kwargs): + state["session"] = kwargs + + def append_text(self, text): + state["text"] = text + + def commit(self): + state["committed"] = True + + def finish(self): + state["finished"] = True + callback = state["callback"] + callback.on_event( + { + "type": "response.audio.delta", + "delta": base64.b64encode(b"ID3-test-audio").decode(), + } + ) + callback.on_event({"type": "response.done"}) + callback.on_event({"type": "session.finished"}) + + def close(self): + state["closed"] = True + + with ( + patch("dashscope.audio.qwen_tts_realtime.QwenTtsRealtime", FakeQwenTtsRealtime), + patch("dashscope.MultiModalConversation.call", side_effect=AssertionError("HTTP TTS should not be used")), + ): + audio = model.text_to_speech("Hello") + + self.assertEqual(audio, b"ID3-test-audio") + self.assertEqual(state["url"], "wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime") + self.assertEqual(state["session"]["mode"], "commit") + self.assertEqual(state["session"]["voice"], "Cherry") + self.assertEqual(state["session"]["audio_format"], "mp3") + self.assertEqual(state["text"], "Hello") + self.assertTrue(state["committed"]) + self.assertTrue(state["finished"]) + self.assertTrue(state["closed"]) + + def test_qwen_audio_tts_uses_audio_generation_endpoint(self): + from models_provider.impl.aliyun_bai_lian_model_provider.model.tts import AliyunBaiLianTextToSpeech + + model = AliyunBaiLianTextToSpeech( + api_key="test-key", + base_url="https://workspace.cn-beijing.maas.aliyuncs.com/api/v1", + model="qwen-audio-3.1-tts-next", + params={"voice": "Cherry", "speech_rate": 0.8, "format": "mp3"}, + ) + response = Mock(status_code=200) + response.json.return_value = {"output": {"audio": {"url": "https://example.com/audio.mp3"}}} + download = Mock(content=b"ID3-qwen-audio") + + with ( + patch("requests.post", return_value=response) as post, + patch("requests.get", return_value=download) as get, + patch("dashscope.MultiModalConversation.call", side_effect=AssertionError("wrong endpoint")), + ): + audio = model.text_to_speech("Hello") + + self.assertEqual(audio, b"ID3-qwen-audio") + post.assert_called_once_with( + "https://workspace.cn-beijing.maas.aliyuncs.com/api/v1/services/audio/tts/SpeechSynthesizer", + headers={"Authorization": "Bearer test-key", "Content-Type": "application/json"}, + json={ + "model": "qwen-audio-3.1-tts-next", + "input": {"text_prompt": "Hello", "rate": 0.8, "format": "mp3"}, + }, + timeout=300, + ) + get.assert_called_once_with("https://example.com/audio.mp3", timeout=30) + + def test_qwen_audio_tts_plus_uses_text_and_voice(self): + from models_provider.impl.aliyun_bai_lian_model_provider.model.tts import AliyunBaiLianTextToSpeech + + model = AliyunBaiLianTextToSpeech( + api_key="test-key", + base_url="https://workspace.cn-beijing.maas.aliyuncs.com/api/v1", + model="qwen-audio-3.1-tts-plus", + params={"voice": "longanhuan_v3.1", "format": "wav", "sample_rate": 24000}, + ) + response = Mock(status_code=200) + response.json.return_value = {"output": {"audio": {"url": "https://example.com/audio.wav"}}} + download = Mock(content=b"RIFF-qwen-audio") + + with ( + patch("requests.post", return_value=response) as post, + patch("requests.get", return_value=download), + patch("dashscope.MultiModalConversation.call", side_effect=AssertionError("wrong endpoint")), + ): + audio = model.text_to_speech("Hello") + + self.assertEqual(audio, b"RIFF-qwen-audio") + post.assert_called_once_with( + "https://workspace.cn-beijing.maas.aliyuncs.com/api/v1/services/audio/tts/SpeechSynthesizer", + headers={"Authorization": "Bearer test-key", "Content-Type": "application/json"}, + json={ + "model": "qwen-audio-3.1-tts-plus", + "input": { + "text": "Hello", + "voice": "longanhuan_v3.1", + "format": "wav", + "sample_rate": 24000, + }, + }, + timeout=300, + ) + + def test_qwen_audio_tts_next_check_auth_uses_audio_generation_prompt(self): + from models_provider.impl.aliyun_bai_lian_model_provider.model.tts import AliyunBaiLianTextToSpeech + + model = AliyunBaiLianTextToSpeech( + api_key="test-key", + base_url="https://workspace.cn-beijing.maas.aliyuncs.com/api/v1", + model="qwen-audio-3.1-tts-next", + params={}, + ) + + with patch.object(AliyunBaiLianTextToSpeech, "text_to_speech") as text_to_speech: + model.check_auth() + + text_to_speech.assert_called_once_with("一位女性清晰地说：“你好，欢迎使用。”") + + def test_qwen_audio_tts_next_params_omit_unused_voice(self): + from models_provider.impl.aliyun_bai_lian_model_provider.credential.tts import AliyunBaiLianTTSModelCredential + + form = AliyunBaiLianTTSModelCredential().get_model_params_setting_form("qwen-audio-3.1-tts-next") + + self.assertEqual(form.get_default_form_data(), {"speech_rate": 1}) + + def test_qwen_audio_tts_plus_uses_matching_default_voice(self): + from models_provider.impl.aliyun_bai_lian_model_provider.credential.tts import AliyunBaiLianTTSModelCredential + + form = AliyunBaiLianTTSModelCredential().get_model_params_setting_form("qwen-audio-3.1-tts-plus") + + self.assertEqual(form.get_default_form_data(), {"voice": "longanhuan_v3.1", "speech_rate": 1}) + + def test_qwen_realtime_tts_reports_server_error_event(self): + from models_provider.impl.aliyun_bai_lian_model_provider.model.tts import AliyunBaiLianTextToSpeech + + model = AliyunBaiLianTextToSpeech( + api_key="invalid-key", + base_url="https://dashscope.aliyuncs.com/api/v1", + model="qwen3-tts-flash-realtime", + params={}, + ) + + class FakeQwenTtsRealtime: + def __init__(self, model, callback, url): + self.callback = callback + + def connect(self): + pass + + def update_session(self, **kwargs): + pass + + def append_text(self, text): + pass + + def commit(self): + pass + + def finish(self): + self.callback.on_event( + { + "type": "error", + "error": {"code": "InvalidApiKey", "message": "Invalid API-key provided."}, + } + ) + + def close(self): + pass + + with ( + patch("dashscope.audio.qwen_tts_realtime.QwenTtsRealtime", FakeQwenTtsRealtime), + patch("dashscope.MultiModalConversation.call", side_effect=AssertionError("HTTP TTS should not be used")), + ): + with self.assertRaisesRegex( + Exception, + r"code: InvalidApiKey, message: Invalid API-key provided\.", + ): + model.text_to_speech("Hello") + + +if __name__ == "__main__": + unittest.main()