From 03ba376d38656dbdc1e2ce5cff6af63864650966 Mon Sep 17 00:00:00 2001 From: JosephTian876 <204833675+JosephTian876@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:05:10 +0800 Subject: [PATCH 1/3] feat(provider): support native video input modality MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ProviderRequest.video_urls so providers that declare the ideo modality receive the video itself instead of only a text placeholder. Fixes two failure modes that a video block would otherwise introduce: - gemini_source._prepare_conversation dispatched user content with a catch-all lse that read item["audio_url"]. A video_url block raised KeyError outside the try that guards the API call, and because the block was persisted the error repeated on every later request. The branch is now exhaustive and unknown block types degrade to text. - assemble_context output is replayed from history on every request, so video blocks are now flagged _no_save and dropped when the message is written back. Video is opt-in: an empty modalities list keeps its existing "supports everything" meaning for text/image/audio but not for video, so existing configurations are unaffected. --- astrbot/core/agent/message.py | 18 +- .../agent/runners/tool_loop_agent_runner.py | 17 +- astrbot/core/astr_main_agent.py | 5 + astrbot/core/config/default.py | 4 +- astrbot/core/provider/entities.py | 47 +++- astrbot/core/provider/modalities.py | 17 +- .../core/provider/sources/gemini_source.py | 74 +++++- .../provider/ProviderChatCompletionPanel.vue | 2 + .../provider/ProviderModelsPanel.vue | 13 + .../src/composables/useProviderSources.ts | 11 + .../i18n/locales/en-US/features/provider.json | 1 + .../i18n/locales/ja-JP/features/provider.json | 1 + .../i18n/locales/ru-RU/features/provider.json | 1 + .../i18n/locales/zh-CN/features/provider.json | 1 + dashboard/src/utils/providerMetadata.ts | 7 + dashboard/src/views/ProviderPage.vue | 2 + tests/test_video_input_modality.py | 242 ++++++++++++++++++ 17 files changed, 452 insertions(+), 11 deletions(-) create mode 100644 tests/test_video_input_modality.py diff --git a/astrbot/core/agent/message.py b/astrbot/core/agent/message.py index 4292f4c04e6..94e457ca69c 100644 --- a/astrbot/core/agent/message.py +++ b/astrbot/core/agent/message.py @@ -21,7 +21,7 @@ class ContentPart(BaseModel): __content_part_registry: ClassVar[dict[str, type["ContentPart"]]] = {} - type: Literal["text", "think", "image_url", "audio_url"] + type: Literal["text", "think", "image_url", "audio_url", "video_url"] _no_save: bool = PrivateAttr(default=False) def __init_subclass__(cls, **kwargs: Any) -> None: @@ -141,6 +141,22 @@ class AudioURL(BaseModel): audio_url: AudioURL +class VideoURLPart(ContentPart): + """ + >>> VideoURLPart(video_url=VideoURLPart.VideoURL(url="https://example.com/clip.mp4")).model_dump() + {'type': 'video_url', 'video_url': {'url': 'https://example.com/clip.mp4', 'id': None}} + """ + + class VideoURL(BaseModel): + url: str + """The URL of the video, can be data URI scheme like `data:video/mp4;base64,...`.""" + id: str | None = None + """The ID of the video, to allow LLMs to distinguish different videos.""" + + type: str = "video_url" + video_url: VideoURL + + class ToolCall(BaseModel): """ A tool call requested by the assistant. diff --git a/astrbot/core/agent/runners/tool_loop_agent_runner.py b/astrbot/core/agent/runners/tool_loop_agent_runner.py index f8418fe4205..8ed30b67dfe 100644 --- a/astrbot/core/agent/runners/tool_loop_agent_runner.py +++ b/astrbot/core/agent/runners/tool_loop_agent_runner.py @@ -317,6 +317,7 @@ async def reset( request.prompt is not None or request.image_urls or request.audio_urls + or request.video_urls or request.extra_user_content_parts ): m = await self._assemble_request_context_for_provider(request) @@ -342,17 +343,26 @@ async def _assemble_request_context_for_provider( ) -> dict[str, T.Any]: modalities = self.provider.provider_config.get("modalities", None) if not modalities: # Unconfigured (None or empty list) defaults to support all modalities for backward compatibility - return await request.assemble_context() + # Video is excluded from that default. It postdates the "supports + # everything" convention and is not universally supported, so a + # provider that never opted in would fail the whole request rather + # than degrade. Video is therefore strictly opt-in via the + # `modalities` list. + if not request.video_urls: + return await request.assemble_context() + return await replace(request, video_urls=[]).assemble_context() supports_image = "image" in modalities supports_audio = "audio" in modalities - if supports_image and supports_audio: + supports_video = "video" in modalities + if supports_image and supports_audio and supports_video: return await request.assemble_context() adjusted_request = replace( request, image_urls=request.image_urls if supports_image else [], audio_urls=request.audio_urls if supports_audio else [], + video_urls=request.video_urls if supports_video else [], ) context = await adjusted_request.assemble_context() content = context.get("content") @@ -369,6 +379,9 @@ async def _assemble_request_context_for_provider( if not supports_audio: for _ in request.audio_urls: content_blocks.append({"type": "text", "text": "[Audio]"}) + if not supports_video: + for _ in request.video_urls: + content_blocks.append({"type": "text", "text": "[Video]"}) return {"role": "user", "content": content_blocks} diff --git a/astrbot/core/astr_main_agent.py b/astrbot/core/astr_main_agent.py index 59183b8a613..a5390e387ca 100644 --- a/astrbot/core/astr_main_agent.py +++ b/astrbot/core/astr_main_agent.py @@ -877,6 +877,10 @@ async def _append_video_attachment( else: text = f"[Video Attachment: name {video_name}, path {video_path}]" + # Collect the resolved path so providers that declare the `video` modality + # receive the video itself. The text part above is kept for every other + # provider, and for the persisted history where the video block is dropped. + req.video_urls.append(video_path) req.extra_user_content_parts.append(TextPart(text=text)) @@ -1821,6 +1825,7 @@ async def build_main_agent( ) req.image_urls = normalize_and_dedupe_strings(req.image_urls) req.audio_urls = normalize_and_dedupe_strings(req.audio_urls) + req.video_urls = normalize_and_dedupe_strings(req.video_urls) if config.file_extract_enabled: try: diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index 150804573a8..2a8abdad8f9 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -2248,8 +2248,8 @@ def get_local_permission_defaults(system: str | None = None) -> dict: "description": "模型能力", "type": "list", "items": {"type": "string"}, - "options": ["text", "image", "audio", "tool_use"], - "labels": ["文本", "图像", "音频", "工具使用"], + "options": ["text", "image", "audio", "video", "tool_use"], + "labels": ["文本", "图像", "音频", "视频", "工具使用"], "render_type": "checkbox", "hint": "模型支持的模态及能力。", }, diff --git a/astrbot/core/provider/entities.py b/astrbot/core/provider/entities.py index 382932b8579..59fc2619586 100644 --- a/astrbot/core/provider/entities.py +++ b/astrbot/core/provider/entities.py @@ -101,6 +101,8 @@ class ProviderRequest: """图片 URL 列表""" audio_urls: list[str] = field(default_factory=list) """音频 URL 列表，也支持本地路径""" + video_urls: list[str] = field(default_factory=list) + """Video URL list. Local paths are supported as well.""" extra_user_content_parts: list[ContentPart] = field(default_factory=list) """额外的用户消息内容部分列表，用于在用户消息后添加额外的内容块（如系统提醒、指令等）。支持 dict 或 ContentPart 对象""" func_tool: ToolSet | None = None @@ -124,6 +126,7 @@ def __repr__(self) -> str: f"ProviderRequest(prompt={self.prompt}, session_id={self.session_id}, " f"image_count={len(self.image_urls or [])}, " f"audio_count={len(self.audio_urls or [])}, " + f"video_count={len(self.video_urls or [])}, " f"func_tool={self.func_tool}, " f"contexts={self._print_friendly_context()}, " f"system_prompt={self.system_prompt}, " @@ -146,7 +149,8 @@ def _print_friendly_context(self): if not self.contexts: return ( f"prompt: {self.prompt}, image_count: {len(self.image_urls or [])}, " - f"audio_count: {len(self.audio_urls or [])}" + f"audio_count: {len(self.audio_urls or [])}, " + f"video_count: {len(self.video_urls or [])}" ) result_parts = [] @@ -163,6 +167,7 @@ def _print_friendly_context(self): msg_parts = [] image_count = 0 audio_count = 0 + video_count = 0 for item in content: item_type = item.get("type", "") @@ -173,6 +178,8 @@ def _print_friendly_context(self): image_count += 1 elif item_type == "audio_url": audio_count += 1 + elif item_type == "video_url": + video_count += 1 if image_count > 0: if msg_parts: @@ -184,6 +191,11 @@ def _print_friendly_context(self): msg_parts.append(f"[+{audio_count} audios]") else: msg_parts.append(f"[{audio_count} audios]") + if video_count > 0: + if msg_parts: + msg_parts.append(f"[+{video_count} videos]") + else: + msg_parts.append(f"[{video_count} videos]") result_parts.append(f"{role}: {''.join(msg_parts)}") @@ -208,6 +220,9 @@ async def assemble_context(self) -> dict: elif self.audio_urls: # 如果没有文本但有音频，添加占位文本 content_blocks.append({"type": "text", "text": "[音频]"}) + elif self.video_urls: + # If there is no text but there is video, add a placeholder text. + content_blocks.append({"type": "text", "text": "[Video]"}) # 2. 额外的内容块（系统提醒、指令等） if self.extra_user_content_parts: @@ -299,6 +314,35 @@ async def assemble_context(self) -> dict: }, ) + # 5. Video content. The block is flagged `_no_save` so that the + # multi-megabyte data URI is dropped when the message is written back to + # the conversation history; otherwise it would be replayed on every later + # request in the session. Providers still receive it, because they + # serialize messages with `model_dump()`, which does not emit `_no_save`. + if self.video_urls: + for video_url in self.video_urls: + try: + video_data = await MediaResolver( + video_url, + media_type="video", + default_suffix=".mp4", + ).to_base64_data(strict=True) + except Exception as exc: + logger.warning( + "Video preprocessing failed; skipping it. Error: %s", exc + ) + continue + if not video_data: + logger.warning("Video preprocessing returned no data; skipping it.") + continue + content_blocks.append( + { + "type": "video_url", + "video_url": {"url": video_data.to_data_url()}, + "_no_save": True, + }, + ) + if image_capture_failed and not any( block.get("type") != "text" or block.get("text", "").strip() for block in content_blocks @@ -312,6 +356,7 @@ async def assemble_context(self) -> dict: and not self.extra_user_content_parts and not self.image_urls and not self.audio_urls + and not self.video_urls ): return {"role": "user", "content": content_blocks[0]["text"]} diff --git a/astrbot/core/provider/modalities.py b/astrbot/core/provider/modalities.py index 66ac74e9b72..44a3f502e26 100644 --- a/astrbot/core/provider/modalities.py +++ b/astrbot/core/provider/modalities.py @@ -13,6 +13,7 @@ class ContextSanitizeStats: fixed_image_blocks: int = 0 fixed_audio_blocks: int = 0 + fixed_video_blocks: int = 0 fixed_tool_messages: int = 0 removed_tool_calls: int = 0 @@ -21,6 +22,7 @@ def changed(self) -> bool: return bool( self.fixed_image_blocks or self.fixed_audio_blocks + or self.fixed_video_blocks or self.fixed_tool_messages or self.removed_tool_calls ) @@ -50,8 +52,9 @@ def sanitize_contexts_by_modalities( supports_image = "image" in modalities supports_audio = "audio" in modalities + supports_video = "video" in modalities supports_tool_use = "tool_use" in modalities - if supports_image and supports_audio and supports_tool_use: + if supports_image and supports_audio and supports_video and supports_tool_use: copied_contexts = [] for msg in contexts: copied_msg = _message_to_dict(msg) @@ -83,7 +86,7 @@ def sanitize_contexts_by_modalities( msg.pop("tool_calls", None) msg.pop("tool_call_id", None) - if not supports_image or not supports_audio: + if not supports_image or not supports_audio or not supports_video: content = msg.get("content") if isinstance(content, list): filtered_parts: list[Any] = [] @@ -104,6 +107,11 @@ def sanitize_contexts_by_modalities( stats.fixed_audio_blocks += 1 filtered_parts.append({"type": "text", "text": "[Audio]"}) continue + if not supports_video and part_type == "video_url": + removed_any_multimodal = True + stats.fixed_video_blocks += 1 + filtered_parts.append({"type": "text", "text": "[Video]"}) + continue filtered_parts.append(part) if removed_any_multimodal: msg["content"] = filtered_parts @@ -136,6 +144,8 @@ def _tool_result_placeholder(content: Any) -> str: text_parts.append("[Image]") elif part_type in {"audio_url", "input_audio"}: text_parts.append("[Audio]") + elif part_type == "video_url": + text_parts.append("[Video]") content_text = "\n".join(part for part in text_parts if part).strip() else: content_text = "" @@ -149,10 +159,11 @@ def log_context_sanitize_stats(stats: ContextSanitizeStats) -> None: return logger.debug( "context modality fix applied: " - "fixed_image_blocks=%s, fixed_audio_blocks=%s, " + "fixed_image_blocks=%s, fixed_audio_blocks=%s, fixed_video_blocks=%s, " "fixed_tool_messages=%s, removed_tool_calls=%s", stats.fixed_image_blocks, stats.fixed_audio_blocks, + stats.fixed_video_blocks, stats.fixed_tool_messages, stats.removed_tool_calls, ) diff --git a/astrbot/core/provider/sources/gemini_source.py b/astrbot/core/provider/sources/gemini_source.py index 0d8df6af99b..63151aff74a 100644 --- a/astrbot/core/provider/sources/gemini_source.py +++ b/astrbot/core/provider/sources/gemini_source.py @@ -15,7 +15,13 @@ import astrbot.core.message.components as Comp from astrbot import logger from astrbot.api.provider import Provider -from astrbot.core.agent.message import AudioURLPart, ContentPart, ImageURLPart, TextPart +from astrbot.core.agent.message import ( + AudioURLPart, + ContentPart, + ImageURLPart, + TextPart, + VideoURLPart, +) from astrbot.core.exceptions import EmptyModelOutputError from astrbot.core.message.message_event_result import MessageChain from astrbot.core.provider.entities import LLMResponse, TokenUsage @@ -407,6 +413,22 @@ def process_audio_url(audio_url_dict: dict) -> types.Part: audio_bytes = base64.b64decode(url.split(",", 1)[1]) return types.Part.from_bytes(data=audio_bytes, mime_type=mime_type) + async def process_video_url(video_url_dict: dict) -> types.Part: + url = video_url_dict["url"] + video_data = await resolve_media_ref_to_base64_data( + url, + media_type="video", + strict=True, + ) + if video_data is None: + raise ValueError( + f"Failed to resolve Gemini history video: {describe_media_ref(url)}" + ) + return types.Part.from_bytes( + data=base64.b64decode(video_data.base64_data), + mime_type=video_data.mime_type, + ) + def append_or_extend( contents: list[types.Content], part: list[types.Part], @@ -430,8 +452,16 @@ def append_or_extend( parts.append(types.Part.from_text(text=item["text"] or " ")) elif item["type"] == "image_url": parts.append(await process_image_url(item["image_url"])) - else: + elif item["type"] == "audio_url": parts.append(process_audio_url(item["audio_url"])) + elif item["type"] == "video_url": + parts.append(await process_video_url(item["video_url"])) + else: + # Never raise for an unknown block type here. This + # message is replayed from the session history on + # every later request, so an exception would keep + # failing long after the original turn. + parts.append(create_text_part(f"[{item['type']}]")) else: parts = [create_text_part(content)] append_or_extend(gemini_contents, parts, types.UserContent) @@ -944,6 +974,7 @@ async def text_chat( session_id=None, image_urls=None, audio_urls=None, + video_urls=None, func_tool=None, contexts=None, system_prompt=None, @@ -963,6 +994,7 @@ async def text_chat( prompt or "", image_urls, audio_urls, + video_urls, extra_user_content_parts, ) context_query = self._ensure_message_to_dicts(contexts) @@ -1016,6 +1048,7 @@ async def text_chat_stream( session_id=None, image_urls=None, audio_urls=None, + video_urls=None, func_tool=None, contexts=None, system_prompt=None, @@ -1035,6 +1068,7 @@ async def text_chat_stream( prompt or "", image_urls, audio_urls, + video_urls, extra_user_content_parts, ) context_query = self._ensure_message_to_dicts(contexts) @@ -1113,6 +1147,7 @@ async def assemble_context( text: str, image_urls: list[str] | None = None, audio_urls: list[str] | None = None, + video_urls: list[str] | None = None, extra_user_content_parts: list[ContentPart] | None = None, ): """组装上下文。""" @@ -1151,6 +1186,27 @@ async def resolve_audio_part(audio_path: str) -> dict | None: "audio_url": {"url": audio_data.to_data_url()}, } + async def resolve_video_part(video_ref: str) -> dict | None: + try: + video_data = await resolve_media_ref_to_base64_data( + video_ref, + media_type="video", + strict=True, + ) + except Exception as exc: + logger.warning( + "Video preprocessing failed; ignoring it. Error: %s", exc + ) + return None + + if not video_data: + logger.warning("Video preprocessing returned no data; ignoring it.") + return None + return { + "type": "video_url", + "video_url": {"url": video_data.to_data_url()}, + } + # 构建内容块列表 content_blocks = [] @@ -1162,6 +1218,9 @@ async def resolve_audio_part(audio_path: str) -> dict | None: content_blocks.append({"type": "text", "text": "[Image]"}) elif audio_urls: content_blocks.append({"type": "text", "text": "[Audio]"}) + elif video_urls: + # If there is no text but there is video, add a placeholder text. + content_blocks.append({"type": "text", "text": "[Video]"}) elif extra_user_content_parts: # 如果只有额外内容块，也需要添加占位文本 content_blocks.append({"type": "text", "text": " "}) @@ -1179,6 +1238,10 @@ async def resolve_audio_part(audio_path: str) -> dict | None: audio_part = await resolve_audio_part(part.audio_url.url) if audio_part: content_blocks.append(audio_part) + elif isinstance(part, VideoURLPart): + video_part = await resolve_video_part(part.video_url.url) + if video_part: + content_blocks.append(video_part) else: raise ValueError( f"Unsupported extra content part type: {type(part)}" @@ -1197,12 +1260,19 @@ async def resolve_audio_part(audio_path: str) -> dict | None: if audio_part: content_blocks.append(audio_part) + if video_urls: + for video_ref in video_urls: + video_part = await resolve_video_part(video_ref) + if video_part: + content_blocks.append(video_part) + # 如果只有主文本且没有额外内容块和图片，返回简单格式以保持向后兼容 if ( text and not extra_user_content_parts and not image_urls and not audio_urls + and not video_urls and len(content_blocks) == 1 and content_blocks[0]["type"] == "text" ): diff --git a/dashboard/src/components/provider/ProviderChatCompletionPanel.vue b/dashboard/src/components/provider/ProviderChatCompletionPanel.vue index 2500782e611..a6c334db80f 100644 --- a/dashboard/src/components/provider/ProviderChatCompletionPanel.vue +++ b/dashboard/src/components/provider/ProviderChatCompletionPanel.vue @@ -98,6 +98,7 @@ :is-source-modified="isSourceModified" :supports-image-input="supportsImageInput" :supports-audio-input="supportsAudioInput" + :supports-video-input="supportsVideoInput" :supports-tool-call="supportsToolCall" :supports-reasoning="supportsReasoning" :format-context-limit="formatContextLimit" @@ -248,6 +249,7 @@ const { getSourceDisplayName, supportsImageInput, supportsAudioInput, + supportsVideoInput, supportsToolCall, supportsReasoning, formatContextLimit, diff --git a/dashboard/src/components/provider/ProviderModelsPanel.vue b/dashboard/src/components/provider/ProviderModelsPanel.vue index 9826e1cb7ea..70ebead4451 100644 --- a/dashboard/src/components/provider/ProviderModelsPanel.vue +++ b/dashboard/src/components/provider/ProviderModelsPanel.vue @@ -296,6 +296,10 @@ const props = defineProps({ type: Function, required: true }, + supportsVideoInput: { + type: Function, + required: true + }, supportsToolCall: { type: Function, required: true @@ -367,6 +371,15 @@ const capabilityBadges = (entry) => { enabled: !isConfigured || modalities.includes('audio'), label: props.tm('models.metadata.audio') }, + { + key: 'video', + icon: 'mdi-video-outline', + supported: props.supportsVideoInput(metadata), + // Video is opt-in on the backend, so an unconfigured provider must not + // report it as enabled the way image/audio do. + enabled: modalities.includes('video'), + label: props.tm('models.metadata.video') + }, { key: 'tool_use', icon: 'mdi-wrench-outline', diff --git a/dashboard/src/composables/useProviderSources.ts b/dashboard/src/composables/useProviderSources.ts index aea46f37083..b53e9d5d9ef 100644 --- a/dashboard/src/composables/useProviderSources.ts +++ b/dashboard/src/composables/useProviderSources.ts @@ -365,6 +365,11 @@ export function useProviderSources(options: UseProviderSourcesOptions) { return inputs.includes('audio') } + function supportsVideoInput(meta: any) { + const inputs = meta?.modalities?.input || [] + return inputs.includes('video') + } + function supportsToolCall(meta: any) { return Boolean(meta?.tool_call) } @@ -648,6 +653,11 @@ export function useProviderSources(options: UseProviderSourcesOptions) { if (supportsAudioInput(metadata)) { modalities.push('audio') } + // Video is opt-in: only enable it when the model metadata declares video + // input, never as part of the "assume everything" fallback above. + if (supportsVideoInput(metadata)) { + modalities.push('video') + } if (supportsToolCall(metadata)) { modalities.push('tool_use') } @@ -831,6 +841,7 @@ export function useProviderSources(options: UseProviderSourcesOptions) { getModelMetadata, supportsImageInput, supportsAudioInput, + supportsVideoInput, supportsToolCall, supportsReasoning, formatContextLimit, diff --git a/dashboard/src/i18n/locales/en-US/features/provider.json b/dashboard/src/i18n/locales/en-US/features/provider.json index aad194fcd6f..5bd1cfc5f9b 100644 --- a/dashboard/src/i18n/locales/en-US/features/provider.json +++ b/dashboard/src/i18n/locales/en-US/features/provider.json @@ -165,6 +165,7 @@ "metadata": { "image": "Image input", "audio": "Audio input", + "video": "Video input", "toolUse": "Tool use", "reasoning": "Reasoning", "context": "{tokens} context window", diff --git a/dashboard/src/i18n/locales/ja-JP/features/provider.json b/dashboard/src/i18n/locales/ja-JP/features/provider.json index ef95b6b1e1a..0086bf16b55 100644 --- a/dashboard/src/i18n/locales/ja-JP/features/provider.json +++ b/dashboard/src/i18n/locales/ja-JP/features/provider.json @@ -165,6 +165,7 @@ "metadata": { "image": "画像入力", "audio": "音声入力", + "video": "動画入力", "toolUse": "ツール使用", "reasoning": "推論", "context": "{tokens} トークンのコンテキストウィンドウ", diff --git a/dashboard/src/i18n/locales/ru-RU/features/provider.json b/dashboard/src/i18n/locales/ru-RU/features/provider.json index 3a8dbc7bbe8..8d9282dfab4 100644 --- a/dashboard/src/i18n/locales/ru-RU/features/provider.json +++ b/dashboard/src/i18n/locales/ru-RU/features/provider.json @@ -165,6 +165,7 @@ "metadata": { "image": "Ввод изображений", "audio": "Ввод аудио", + "video": "Ввод видео", "toolUse": "Использование инструментов", "reasoning": "Рассуждение", "context": "Контекстное окно {tokens}", diff --git a/dashboard/src/i18n/locales/zh-CN/features/provider.json b/dashboard/src/i18n/locales/zh-CN/features/provider.json index 0818e442d9c..84d69cb6648 100644 --- a/dashboard/src/i18n/locales/zh-CN/features/provider.json +++ b/dashboard/src/i18n/locales/zh-CN/features/provider.json @@ -165,6 +165,7 @@ "metadata": { "image": "图像输入", "audio": "音频输入", + "video": "视频输入", "toolUse": "工具使用", "reasoning": "推理", "context": "{tokens} 上下文窗口", diff --git a/dashboard/src/utils/providerMetadata.ts b/dashboard/src/utils/providerMetadata.ts index e5fe7e2e51b..9e37a135354 100644 --- a/dashboard/src/utils/providerMetadata.ts +++ b/dashboard/src/utils/providerMetadata.ts @@ -65,6 +65,13 @@ export function providerCapabilityBadges( enabled: modalities.includes('audio'), label: tm('models.metadata.audio') }, + { + key: 'video', + icon: 'mdi-video-outline', + supported: inputs.includes('video'), + enabled: modalities.includes('video'), + label: tm('models.metadata.video') + }, { key: 'tool_use', icon: 'mdi-wrench-outline', diff --git a/dashboard/src/views/ProviderPage.vue b/dashboard/src/views/ProviderPage.vue index 5065bc8afdf..c2f92796600 100644 --- a/dashboard/src/views/ProviderPage.vue +++ b/dashboard/src/views/ProviderPage.vue @@ -138,6 +138,7 @@ :is-source-modified="isSourceModified" :supports-image-input="supportsImageInput" :supports-audio-input="supportsAudioInput" + :supports-video-input="supportsVideoInput" :supports-tool-call="supportsToolCall" :supports-reasoning="supportsReasoning" :format-context-limit="formatContextLimit" @@ -364,6 +365,7 @@ const { getSourceDisplayName, supportsImageInput, supportsAudioInput, + supportsVideoInput, supportsToolCall, supportsReasoning, formatContextLimit, diff --git a/tests/test_video_input_modality.py b/tests/test_video_input_modality.py new file mode 100644 index 00000000000..0b9b0a59780 --- /dev/null +++ b/tests/test_video_input_modality.py @@ -0,0 +1,242 @@ +from types import SimpleNamespace + +import pytest + +from astrbot.core.agent.message import ( + Message, + TextPart, + VideoURLPart, + bind_checkpoint_messages, + dump_messages_with_checkpoints, +) +from astrbot.core.agent.runners.tool_loop_agent_runner import ToolLoopAgentRunner +from astrbot.core.provider.entities import ProviderRequest +from astrbot.core.provider.modalities import sanitize_contexts_by_modalities +from astrbot.core.provider.sources.gemini_source import ProviderGoogleGenAI + +VIDEO_DATA_URL = "data:video/mp4;base64,AAAAGGZ0eXBtcDQyAAAAAG1wNDJpc29t" + + +def _gemini_provider() -> ProviderGoogleGenAI: + provider = ProviderGoogleGenAI.__new__(ProviderGoogleGenAI) + provider.provider_config = {} + provider.provider_settings = {} + return provider + + +def _runner(modalities) -> ToolLoopAgentRunner: + runner = ToolLoopAgentRunner.__new__(ToolLoopAgentRunner) + runner.provider = SimpleNamespace(provider_config={"modalities": modalities}) + return runner + + +def test_video_url_part_is_registered_and_round_trips(): + """A `video_url` dict must resolve through the ContentPart registry. + + The registry lookup in `ContentPart.__get_pydantic_core_schema__` is + unguarded, so an unregistered type raises `KeyError` the moment the block is + validated from persisted history. + """ + message = Message.model_validate( + { + "role": "user", + "content": [{"type": "video_url", "video_url": {"url": VIDEO_DATA_URL}}], + } + ) + + assert isinstance(message.content[0], VideoURLPart) + assert message.content[0].video_url.url == VIDEO_DATA_URL + + +@pytest.mark.asyncio +async def test_assemble_context_marks_video_block_as_transient(): + request = ProviderRequest(prompt="", video_urls=[VIDEO_DATA_URL]) + + context = await request.assemble_context() + + video_blocks = [ + block for block in context["content"] if block.get("type") == "video_url" + ] + assert len(video_blocks) == 1 + assert video_blocks[0]["_no_save"] is True + + +@pytest.mark.asyncio +async def test_video_block_is_not_written_to_history(): + """The video payload must not be persisted and replayed on later requests.""" + request = ProviderRequest(prompt="look", video_urls=[VIDEO_DATA_URL]) + context = await request.assemble_context() + + messages = [Message.model_validate(context)] + dumped = dump_messages_with_checkpoints(messages) + + persisted_types = [ + part["type"] + for part in dumped[0]["content"] + if isinstance(part, dict) + ] + assert "video_url" not in persisted_types + assert "text" in persisted_types + + +@pytest.mark.asyncio +async def test_transient_video_reaches_provider_but_not_history(): + """Prove both halves of the design in one place. + + Providers serialize messages with `model_dump()`, which does not emit + `_no_save`, so the video still reaches the provider payload. Persistence goes + through `dump_messages_with_checkpoints`, which drops it. + """ + request = ProviderRequest(prompt="look", video_urls=[VIDEO_DATA_URL]) + context = await request.assemble_context() + message = Message.model_validate(context) + + assert message.content[1]._no_save is True + + provider_payload = message.model_dump() + assert any(part["type"] == "video_url" for part in provider_payload["content"]) + + persisted = dump_messages_with_checkpoints([message]) + assert not any(part["type"] == "video_url" for part in persisted[0]["content"]) + + +@pytest.mark.asyncio +async def test_gemini_accepts_video_url_block(): + provider = _gemini_provider() + payloads = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe this"}, + {"type": "video_url", "video_url": {"url": VIDEO_DATA_URL}}, + ], + } + ] + } + + contents = await provider._prepare_conversation(payloads) + + parts = contents[0].parts + assert len(parts) == 2 + assert parts[1].inline_data is not None + assert parts[1].inline_data.mime_type == "video/mp4" + + +@pytest.mark.asyncio +async def test_gemini_degrades_unknown_block_type_instead_of_raising(): + """An unknown block must not raise. + + The message is replayed from history on every later request, so raising here + would keep failing long after the turn that produced it. + """ + provider = _gemini_provider() + payloads = { + "messages": [ + { + "role": "user", + "content": [{"type": "some_future_block", "payload": "x"}], + } + ] + } + + contents = await provider._prepare_conversation(payloads) + + assert contents[0].parts[0].text == "[some_future_block]" + + +@pytest.mark.asyncio +async def test_video_is_opt_in_for_providers_without_modalities(): + """An unconfigured provider must not receive video. + + An empty or missing `modalities` list means "supports everything" for + backward compatibility, but video postdates that convention and is not + universally supported. + """ + runner = _runner([]) + request = ProviderRequest(prompt="look", video_urls=[VIDEO_DATA_URL]) + + context = await runner._assemble_request_context_for_provider(request) + + # With the video removed this collapses to the plain-text form, which is the + # pre-existing behaviour for a text-only request. + content = context["content"] + if isinstance(content, str): + assert "video_url" not in content + else: + assert not any(block.get("type") == "video_url" for block in content) + + +@pytest.mark.asyncio +async def test_video_is_sent_when_declared(): + runner = _runner(["text", "image", "audio", "video", "tool_use"]) + request = ProviderRequest(prompt="look", video_urls=[VIDEO_DATA_URL]) + + context = await runner._assemble_request_context_for_provider(request) + + assert any(block.get("type") == "video_url" for block in context["content"]) + + +@pytest.mark.asyncio +async def test_video_is_replaced_with_placeholder_when_not_declared(): + runner = _runner(["text", "image", "audio", "tool_use"]) + request = ProviderRequest(prompt="look", video_urls=[VIDEO_DATA_URL]) + + context = await runner._assemble_request_context_for_provider(request) + + content = context["content"] + assert not any(block.get("type") == "video_url" for block in content) + assert {"type": "text", "text": "[Video]"} in content + + +def test_modality_sanitizer_degrades_video_to_text(): + contexts = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "video_url", "video_url": {"url": VIDEO_DATA_URL}}, + ], + } + ] + + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text", "image"]) + + assert stats.fixed_video_blocks == 1 + assert {"type": "text", "text": "[Video]"} in sanitized[0]["content"] + + +def test_modality_sanitizer_keeps_video_when_declared(): + contexts = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "video_url", "video_url": {"url": VIDEO_DATA_URL}}, + ], + } + ] + + sanitized, stats = sanitize_contexts_by_modalities( + contexts, ["text", "image", "audio", "video"] + ) + + assert stats.fixed_video_blocks == 0 + assert any(block.get("type") == "video_url" for block in sanitized[0]["content"]) + + +@pytest.mark.asyncio +async def test_video_placeholder_is_added_for_video_only_requests(): + request = ProviderRequest(prompt="", video_urls=[VIDEO_DATA_URL]) + + context = await request.assemble_context() + + assert context["content"][0] == {"type": "text", "text": "[Video]"} + + +def test_text_only_request_is_unchanged(): + """The simple string form must be preserved for text-only requests.""" + request = ProviderRequest(prompt="hello") + + assert request.video_urls == [] + assert TextPart(text="hello").text == "hello" From 3ea857d96bf6a75a4d3ee7fc59009e3053b6e6a1 Mon Sep 17 00:00:00 2001 From: JosephTian876 <204833675+JosephTian876@users.noreply.github.com> Date: Wed, 23 Sep 2026 10:19:58 +0800 Subject: [PATCH 2/3] fix(provider): close two video modality gaps found in review Degrade `video_url` for an unconfigured provider on the fallback path. `sanitize_contexts_by_modalities` treats empty or missing `modalities` as "supports everything" for backward compatibility, but video is strictly opt-in. Since that function also sanitizes an already-assembled context when a request falls back to another provider, an unconfigured fallback could receive a `video_url` block that the primary provider produced, and fail the whole request instead of getting `[Video]`. Handle a `VideoURLPart` passed through `extra_user_content_parts` the same way as `video_urls`: resolve it to a portable data URI and flag it transient. Without that the block was persisted verbatim, so a data-URI video stayed in the conversation history as a multi-megabyte payload and a local path was replayed after its file was gone. A message left with nothing but transient blocks now keeps a text placeholder so it cannot persist as empty content. The unconfigured provider keeps receiving its original context objects when there is no video block to degrade, so the sanitizer is not entered and nothing else about that path changes. --- .../agent/runners/tool_loop_agent_runner.py | 13 +- astrbot/core/provider/entities.py | 43 +++++ astrbot/core/provider/modalities.py | 58 +++++- tests/test_video_input_modality.py | 167 +++++++++++++++++- 4 files changed, 270 insertions(+), 11 deletions(-) diff --git a/astrbot/core/agent/runners/tool_loop_agent_runner.py b/astrbot/core/agent/runners/tool_loop_agent_runner.py index 8ed30b67dfe..0da391d6d9d 100644 --- a/astrbot/core/agent/runners/tool_loop_agent_runner.py +++ b/astrbot/core/agent/runners/tool_loop_agent_runner.py @@ -42,6 +42,7 @@ ToolCallsResult, ) from astrbot.core.provider.modalities import ( + has_video_blocks, log_context_sanitize_stats, sanitize_contexts_by_modalities, ) @@ -676,13 +677,17 @@ def _sanitize_contexts_for_provider( contexts: list[Message] | list[dict[str, T.Any]], ) -> list[Message] | list[dict[str, T.Any]]: modalities = self.provider.provider_config.get("modalities", None) - if ( - not modalities - ): # Unconfigured (None or empty list) defaults to support all modalities + if not modalities and not has_video_blocks(contexts): + # Unconfigured (None or empty list) defaults to supporting every + # modality, so the context objects are handed back untouched. Video + # is the exception: it is strictly opt-in (see + # `_assemble_request_context_for_provider`), and this path also runs + # when a request falls back to another provider, whose context was + # assembled for the primary one and may already carry a `video_url`. return contexts sanitized_contexts, stats = sanitize_contexts_by_modalities( contexts, - self.provider.provider_config.get("modalities", None), + modalities, ) log_context_sanitize_stats(stats) return sanitized_contexts diff --git a/astrbot/core/provider/entities.py b/astrbot/core/provider/entities.py index 59fc2619586..2b7fd42cd85 100644 --- a/astrbot/core/provider/entities.py +++ b/astrbot/core/provider/entities.py @@ -210,6 +210,7 @@ async def assemble_context(self) -> dict: # 构建内容块列表 content_blocks = [] image_capture_failed = False + transient_video_added = False # 1. 用户原始发言（OpenAI 建议：用户发言在前） if self.prompt and self.prompt.strip(): @@ -260,6 +261,40 @@ async def assemble_context(self) -> dict: "url": resolved.to_data_url(), }, } + # A video supplied through `extra_user_content_parts` is handled + # like `video_urls` below: resolve it to a portable data URI and + # flag it transient. Persisting it would replay the whole payload + # on every later request, and a local path may be gone by then. + if isinstance(dumped, dict) and dumped.get("type") == "video_url": + video_url = dumped.get("video_url") + url = video_url.get("url") if isinstance(video_url, dict) else None + if isinstance(url, str) and url: + try: + resolved_video = await MediaResolver( + url, + media_type="video", + default_suffix=".mp4", + ).to_base64_data(strict=True) + except Exception as exc: + logger.warning( + "Video preprocessing failed; skipping it. Error: %s", + exc, + ) + continue + if not resolved_video: + logger.warning( + "Video preprocessing returned no data; skipping it." + ) + continue + dumped = { + **dumped, + "video_url": { + **video_url, + "url": resolved_video.to_data_url(), + }, + "_no_save": True, + } + transient_video_added = True content_blocks.append(dumped) # 3. Read image references without resizing or transcoding. @@ -349,6 +384,14 @@ async def assemble_context(self) -> dict: ): content_blocks = [{"type": "text", "text": "[Image unavailable]"}] + # A message made only of transient blocks would be persisted with an + # empty content list and replayed that way on every later request, so + # keep a text placeholder for the attachment. + if transient_video_added and not any( + not block.get("_no_save") for block in content_blocks + ): + content_blocks.insert(0, {"type": "text", "text": "[Video]"}) + # 只有当只有一个来自 prompt 的文本块且没有额外内容块时，才降级为简单格式以保持向后兼容 if ( len(content_blocks) == 1 diff --git a/astrbot/core/provider/modalities.py b/astrbot/core/provider/modalities.py index 44a3f502e26..b028d7541bd 100644 --- a/astrbot/core/provider/modalities.py +++ b/astrbot/core/provider/modalities.py @@ -36,6 +36,33 @@ def _message_to_dict(message: dict[str, Any] | Message) -> dict[str, Any] | None return None +def has_video_blocks(contexts: Sequence[dict[str, Any] | Message]) -> bool: + """Whether any message in `contexts` carries a `video_url` block. + + Accepts both `Message` objects (whose content holds `ContentPart` instances) + and plain dicts, since callers pass either form. + + Args: + contexts: Messages to inspect. + + Returns: + True as soon as a `video_url` block is found. + """ + for msg in contexts: + content = msg.content if isinstance(msg, Message) else msg.get("content") + if not isinstance(content, list): + continue + for part in content: + part_type = ( + part.get("type") + if isinstance(part, dict) + else getattr(part, "type", None) + ) + if part_type == "video_url": + return True + return False + + def sanitize_contexts_by_modalities( contexts: Sequence[dict[str, Any] | Message], modalities: list[str] | None, @@ -43,12 +70,37 @@ def sanitize_contexts_by_modalities( if not contexts: return [], ContextSanitizeStats() if not modalities or not isinstance(modalities, list): + # An unconfigured provider (None or an empty list) keeps the historical + # "supports everything" default for image/audio/tool_use. Video postdates + # that convention and stays strictly opt-in, so it is degraded even here: + # this function also sanitizes a context that was assembled for another + # provider, which may have produced a `video_url` block because *it* + # declared the video modality. Everything else is copied through + # untouched, exactly as before. copied_contexts = [] + stats = ContextSanitizeStats() for msg in contexts: copied_msg = _message_to_dict(msg) - if copied_msg: - copied_contexts.append(copied_msg) - return copied_contexts, ContextSanitizeStats() + if not copied_msg: + continue + content = copied_msg.get("content") + if isinstance(content, list): + filtered_parts: list[Any] = [] + removed_video = False + for part in content: + if ( + isinstance(part, dict) + and str(part.get("type", "")).lower() == "video_url" + ): + removed_video = True + stats.fixed_video_blocks += 1 + filtered_parts.append({"type": "text", "text": "[Video]"}) + continue + filtered_parts.append(part) + if removed_video: + copied_msg["content"] = filtered_parts + copied_contexts.append(copied_msg) + return copied_contexts, stats supports_image = "image" in modalities supports_audio = "audio" in modalities diff --git a/tests/test_video_input_modality.py b/tests/test_video_input_modality.py index 0b9b0a59780..21cffa93136 100644 --- a/tests/test_video_input_modality.py +++ b/tests/test_video_input_modality.py @@ -6,7 +6,6 @@ Message, TextPart, VideoURLPart, - bind_checkpoint_messages, dump_messages_with_checkpoints, ) from astrbot.core.agent.runners.tool_loop_agent_runner import ToolLoopAgentRunner @@ -71,9 +70,7 @@ async def test_video_block_is_not_written_to_history(): dumped = dump_messages_with_checkpoints(messages) persisted_types = [ - part["type"] - for part in dumped[0]["content"] - if isinstance(part, dict) + part["type"] for part in dumped[0]["content"] if isinstance(part, dict) ] assert "video_url" not in persisted_types assert "text" in persisted_types @@ -240,3 +237,165 @@ def test_text_only_request_is_unchanged(): assert request.video_urls == [] assert TextPart(text="hello").text == "hello" + + +def _video_context() -> list[dict]: + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "video_url", "video_url": {"url": VIDEO_DATA_URL}}, + ], + } + ] + + +@pytest.mark.parametrize("modalities", [[], None]) +def test_unconfigured_sanitizer_degrades_video(modalities): + """Video must stay opt-in on the sanitize path too. + + `sanitize_contexts_by_modalities` also runs on an already-assembled context + when a request falls back to another provider. That context may carry a + `video_url` block produced by the primary provider, so an unconfigured + fallback must not inherit it just because empty modalities means "supports + everything" for the other modalities. + """ + sanitized, stats = sanitize_contexts_by_modalities(_video_context(), modalities) + + assert stats.fixed_video_blocks == 1 + assert stats.changed is True + assert not any(part.get("type") == "video_url" for part in sanitized[0]["content"]) + assert {"type": "text", "text": "[Video]"} in sanitized[0]["content"] + + +def test_unconfigured_sanitizer_leaves_the_rest_of_the_history_alone(): + """Only video is degraded for an unconfigured provider. + + Everything else must be copied through untouched, including messages that + the configured-provider path would filter out. + """ + contexts = [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": ""}, + {"content": "message without a role"}, + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://x/i.png"}}], + }, + ] + + sanitized, stats = sanitize_contexts_by_modalities(contexts, []) + + assert sanitized == contexts + assert stats.changed is False + + +def test_fallback_sanitizer_strips_video_for_unconfigured_provider(): + """The runner's fallback path must apply the same opt-in rule. + + The primary provider may have declared video while the fallback provider is + unconfigured, and the context is sanitized only once the fallback is tried. + """ + runner = _runner([]) + + sanitized = runner._sanitize_contexts_for_provider(_video_context()) + + content = sanitized[0]["content"] + assert not any(part.get("type") == "video_url" for part in content) + assert {"type": "text", "text": "[Video]"} in content + + +def test_unconfigured_provider_keeps_its_context_objects_without_video(): + """The sanitizer is only entered when there is a video block to degrade. + + An unconfigured provider must keep receiving the very same context objects + it always did; converting a plain-text history to dicts is an unrelated + behaviour change. + """ + runner = _runner([]) + contexts = [{"role": "user", "content": "hello"}] + + assert runner._sanitize_contexts_for_provider(contexts) is contexts + + +def test_fallback_sanitizer_strips_video_from_message_objects(): + """The production path hands the sanitizer `Message` objects, not dicts.""" + runner = _runner([]) + message = Message.model_validate(_video_context()[0]) + + sanitized = runner._sanitize_contexts_for_provider([message]) + + content = sanitized[0]["content"] + assert not any(part.get("type") == "video_url" for part in content) + assert {"type": "text", "text": "[Video]"} in content + + +@pytest.mark.asyncio +async def test_extra_video_part_is_resolved_and_marked_transient(): + """A `VideoURLPart` in `extra_user_content_parts` must behave like `video_urls`. + + It has to be resolved to a portable data URI and flagged transient, or the + payload is persisted and a local path is replayed after the file is gone. + """ + request = ProviderRequest( + prompt="look", + extra_user_content_parts=[ + VideoURLPart(video_url=VideoURLPart.VideoURL(url=VIDEO_DATA_URL)) + ], + ) + + context = await request.assemble_context() + + video_blocks = [b for b in context["content"] if b.get("type") == "video_url"] + assert len(video_blocks) == 1 + assert video_blocks[0]["_no_save"] is True + assert video_blocks[0]["video_url"]["url"].startswith("data:video/mp4;base64,") + + +@pytest.mark.asyncio +async def test_extra_video_part_reaches_provider_but_not_history(): + request = ProviderRequest( + prompt="look", + extra_user_content_parts=[ + VideoURLPart(video_url=VideoURLPart.VideoURL(url=VIDEO_DATA_URL)) + ], + ) + message = Message.model_validate(await request.assemble_context()) + + assert any(part["type"] == "video_url" for part in message.model_dump()["content"]) + + persisted = dump_messages_with_checkpoints([message]) + assert not any(part["type"] == "video_url" for part in persisted[0]["content"]) + + +@pytest.mark.asyncio +async def test_extra_video_only_message_keeps_a_text_placeholder(): + """A message of only transient blocks must not persist as empty content.""" + request = ProviderRequest( + prompt="", + extra_user_content_parts=[ + VideoURLPart(video_url=VideoURLPart.VideoURL(url=VIDEO_DATA_URL)) + ], + ) + message = Message.model_validate(await request.assemble_context()) + + persisted = dump_messages_with_checkpoints([message]) + assert not any(part.get("type") == "video_url" for part in persisted[0]["content"]) + assert {"type": "text", "text": "[Video]"} in persisted[0]["content"] + + +@pytest.mark.asyncio +async def test_extra_video_part_is_dropped_when_it_cannot_be_resolved(tmp_path): + request = ProviderRequest( + prompt="look", + extra_user_content_parts=[ + VideoURLPart( + video_url=VideoURLPart.VideoURL(url=str(tmp_path / "missing.mp4")) + ) + ], + ) + + context = await request.assemble_context() + + assert not any(b.get("type") == "video_url" for b in context["content"]) From d2c2c7e9ddafaa6f64f74d374a157570435fb742 Mon Sep 17 00:00:00 2001 From: JosephTian876 <204833675+JosephTian876@users.noreply.github.com> Date: Mon, 12 Oct 2026 21:29:15 +0800 Subject: [PATCH 3/3] fix(provider): degrade VideoURLPart for OpenAI and Anthropic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Providers without native video input fell through to the catch-all lse: raise ValueError(...) in their xtra_user_content_parts loop, so a VideoURLPart raised instead of degrading. Video is opt-in and those APIs cannot accept it, so emit a [Video] text placeholder like every other unsupported modality and never persist the payload. --- astrbot/core/provider/sources/anthropic_source.py | 12 +++++++++++- astrbot/core/provider/sources/openai_source.py | 5 +++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/astrbot/core/provider/sources/anthropic_source.py b/astrbot/core/provider/sources/anthropic_source.py index 20a3a813d5d..a07ce575133 100644 --- a/astrbot/core/provider/sources/anthropic_source.py +++ b/astrbot/core/provider/sources/anthropic_source.py @@ -13,7 +13,13 @@ from astrbot import logger from astrbot.api.provider import Provider -from astrbot.core.agent.message import AudioURLPart, ContentPart, ImageURLPart, TextPart +from astrbot.core.agent.message import ( + AudioURLPart, + ContentPart, + ImageURLPart, + TextPart, + VideoURLPart, +) from astrbot.core.exceptions import EmptyModelOutputError from astrbot.core.provider.entities import LLMResponse, TokenUsage from astrbot.core.provider.func_tool_manager import ToolSet @@ -1023,6 +1029,10 @@ async def resolve_image_url(image_url: str) -> dict | None: content.append(image_dict) elif isinstance(block, AudioURLPart): content.append({"type": "text", "text": "[Audio]"}) + elif isinstance(block, VideoURLPart): + # No native video input here and history is replayed, so + # degrade to a text placeholder instead of resolving base64. + content.append({"type": "text", "text": "[Video]"}) else: raise ValueError(f"不支持的额外内容块类型: {type(block)}") diff --git a/astrbot/core/provider/sources/openai_source.py b/astrbot/core/provider/sources/openai_source.py index 9b0c89d5d28..f4f7534d008 100644 --- a/astrbot/core/provider/sources/openai_source.py +++ b/astrbot/core/provider/sources/openai_source.py @@ -24,6 +24,7 @@ ImageURLPart, Message, TextPart, + VideoURLPart, ) from astrbot.core.agent.tool import ToolSet from astrbot.core.exceptions import EmptyModelOutputError @@ -1420,6 +1421,10 @@ async def assemble_context( audio_part = await self._resolve_audio_part(part.audio_url.url) if audio_part: content_blocks.append(audio_part) + elif isinstance(part, VideoURLPart): + # No native video input here and history is replayed, so + # degrade to a text placeholder instead of resolving base64. + content_blocks.append({"type": "text", "text": "[Video]"}) else: raise ValueError(f"不支持的额外内容块类型: {type(part)}")