diff --git a/astrbot/core/agent/message.py b/astrbot/core/agent/message.py index 4292f4c04e6..94e457ca69c 100644 --- a/astrbot/core/agent/message.py +++ b/astrbot/core/agent/message.py @@ -21,7 +21,7 @@ class ContentPart(BaseModel): __content_part_registry: ClassVar[dict[str, type["ContentPart"]]] = {} - type: Literal["text", "think", "image_url", "audio_url"] + type: Literal["text", "think", "image_url", "audio_url", "video_url"] _no_save: bool = PrivateAttr(default=False) def __init_subclass__(cls, **kwargs: Any) -> None: @@ -141,6 +141,22 @@ class AudioURL(BaseModel): audio_url: AudioURL +class VideoURLPart(ContentPart): + """ + >>> VideoURLPart(video_url=VideoURLPart.VideoURL(url="https://example.com/clip.mp4")).model_dump() + {'type': 'video_url', 'video_url': {'url': 'https://example.com/clip.mp4', 'id': None}} + """ + + class VideoURL(BaseModel): + url: str + """The URL of the video, can be data URI scheme like `data:video/mp4;base64,...`.""" + id: str | None = None + """The ID of the video, to allow LLMs to distinguish different videos.""" + + type: str = "video_url" + video_url: VideoURL + + class ToolCall(BaseModel): """ A tool call requested by the assistant. diff --git a/astrbot/core/agent/runners/tool_loop_agent_runner.py b/astrbot/core/agent/runners/tool_loop_agent_runner.py index f8418fe4205..0da391d6d9d 100644 --- a/astrbot/core/agent/runners/tool_loop_agent_runner.py +++ b/astrbot/core/agent/runners/tool_loop_agent_runner.py @@ -42,6 +42,7 @@ ToolCallsResult, ) from astrbot.core.provider.modalities import ( + has_video_blocks, log_context_sanitize_stats, sanitize_contexts_by_modalities, ) @@ -317,6 +318,7 @@ async def reset( request.prompt is not None or request.image_urls or request.audio_urls + or request.video_urls or request.extra_user_content_parts ): m = await self._assemble_request_context_for_provider(request) @@ -342,17 +344,26 @@ async def _assemble_request_context_for_provider( ) -> dict[str, T.Any]: modalities = self.provider.provider_config.get("modalities", None) if not modalities: # Unconfigured (None or empty list) defaults to support all modalities for backward compatibility - return await request.assemble_context() + # Video is excluded from that default. It postdates the "supports + # everything" convention and is not universally supported, so a + # provider that never opted in would fail the whole request rather + # than degrade. Video is therefore strictly opt-in via the + # `modalities` list. + if not request.video_urls: + return await request.assemble_context() + return await replace(request, video_urls=[]).assemble_context() supports_image = "image" in modalities supports_audio = "audio" in modalities - if supports_image and supports_audio: + supports_video = "video" in modalities + if supports_image and supports_audio and supports_video: return await request.assemble_context() adjusted_request = replace( request, image_urls=request.image_urls if supports_image else [], audio_urls=request.audio_urls if supports_audio else [], + video_urls=request.video_urls if supports_video else [], ) context = await adjusted_request.assemble_context() content = context.get("content") @@ -369,6 +380,9 @@ async def _assemble_request_context_for_provider( if not supports_audio: for _ in request.audio_urls: content_blocks.append({"type": "text", "text": "[Audio]"}) + if not supports_video: + for _ in request.video_urls: + content_blocks.append({"type": "text", "text": "[Video]"}) return {"role": "user", "content": content_blocks} @@ -663,13 +677,17 @@ def _sanitize_contexts_for_provider( contexts: list[Message] | list[dict[str, T.Any]], ) -> list[Message] | list[dict[str, T.Any]]: modalities = self.provider.provider_config.get("modalities", None) - if ( - not modalities - ): # Unconfigured (None or empty list) defaults to support all modalities + if not modalities and not has_video_blocks(contexts): + # Unconfigured (None or empty list) defaults to supporting every + # modality, so the context objects are handed back untouched. Video + # is the exception: it is strictly opt-in (see + # `_assemble_request_context_for_provider`), and this path also runs + # when a request falls back to another provider, whose context was + # assembled for the primary one and may already carry a `video_url`. return contexts sanitized_contexts, stats = sanitize_contexts_by_modalities( contexts, - self.provider.provider_config.get("modalities", None), + modalities, ) log_context_sanitize_stats(stats) return sanitized_contexts diff --git a/astrbot/core/astr_main_agent.py b/astrbot/core/astr_main_agent.py index 59183b8a613..a5390e387ca 100644 --- a/astrbot/core/astr_main_agent.py +++ b/astrbot/core/astr_main_agent.py @@ -877,6 +877,10 @@ async def _append_video_attachment( else: text = f"[Video Attachment: name {video_name}, path {video_path}]" + # Collect the resolved path so providers that declare the `video` modality + # receive the video itself. The text part above is kept for every other + # provider, and for the persisted history where the video block is dropped. + req.video_urls.append(video_path) req.extra_user_content_parts.append(TextPart(text=text)) @@ -1821,6 +1825,7 @@ async def build_main_agent( ) req.image_urls = normalize_and_dedupe_strings(req.image_urls) req.audio_urls = normalize_and_dedupe_strings(req.audio_urls) + req.video_urls = normalize_and_dedupe_strings(req.video_urls) if config.file_extract_enabled: try: diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index 150804573a8..2a8abdad8f9 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -2248,8 +2248,8 @@ def get_local_permission_defaults(system: str | None = None) -> dict: "description": "模型能力", "type": "list", "items": {"type": "string"}, - "options": ["text", "image", "audio", "tool_use"], - "labels": ["文本", "图像", "音频", "工具使用"], + "options": ["text", "image", "audio", "video", "tool_use"], + "labels": ["文本", "图像", "音频", "视频", "工具使用"], "render_type": "checkbox", "hint": "模型支持的模态及能力。", }, diff --git a/astrbot/core/provider/entities.py b/astrbot/core/provider/entities.py index 382932b8579..2b7fd42cd85 100644 --- a/astrbot/core/provider/entities.py +++ b/astrbot/core/provider/entities.py @@ -101,6 +101,8 @@ class ProviderRequest: """图片 URL 列表""" audio_urls: list[str] = field(default_factory=list) """音频 URL 列表,也支持本地路径""" + video_urls: list[str] = field(default_factory=list) + """Video URL list. Local paths are supported as well.""" extra_user_content_parts: list[ContentPart] = field(default_factory=list) """额外的用户消息内容部分列表,用于在用户消息后添加额外的内容块(如系统提醒、指令等)。支持 dict 或 ContentPart 对象""" func_tool: ToolSet | None = None @@ -124,6 +126,7 @@ def __repr__(self) -> str: f"ProviderRequest(prompt={self.prompt}, session_id={self.session_id}, " f"image_count={len(self.image_urls or [])}, " f"audio_count={len(self.audio_urls or [])}, " + f"video_count={len(self.video_urls or [])}, " f"func_tool={self.func_tool}, " f"contexts={self._print_friendly_context()}, " f"system_prompt={self.system_prompt}, " @@ -146,7 +149,8 @@ def _print_friendly_context(self): if not self.contexts: return ( f"prompt: {self.prompt}, image_count: {len(self.image_urls or [])}, " - f"audio_count: {len(self.audio_urls or [])}" + f"audio_count: {len(self.audio_urls or [])}, " + f"video_count: {len(self.video_urls or [])}" ) result_parts = [] @@ -163,6 +167,7 @@ def _print_friendly_context(self): msg_parts = [] image_count = 0 audio_count = 0 + video_count = 0 for item in content: item_type = item.get("type", "") @@ -173,6 +178,8 @@ def _print_friendly_context(self): image_count += 1 elif item_type == "audio_url": audio_count += 1 + elif item_type == "video_url": + video_count += 1 if image_count > 0: if msg_parts: @@ -184,6 +191,11 @@ def _print_friendly_context(self): msg_parts.append(f"[+{audio_count} audios]") else: msg_parts.append(f"[{audio_count} audios]") + if video_count > 0: + if msg_parts: + msg_parts.append(f"[+{video_count} videos]") + else: + msg_parts.append(f"[{video_count} videos]") result_parts.append(f"{role}: {''.join(msg_parts)}") @@ -198,6 +210,7 @@ async def assemble_context(self) -> dict: # 构建内容块列表 content_blocks = [] image_capture_failed = False + transient_video_added = False # 1. 用户原始发言(OpenAI 建议:用户发言在前) if self.prompt and self.prompt.strip(): @@ -208,6 +221,9 @@ async def assemble_context(self) -> dict: elif self.audio_urls: # 如果没有文本但有音频,添加占位文本 content_blocks.append({"type": "text", "text": "[音频]"}) + elif self.video_urls: + # If there is no text but there is video, add a placeholder text. + content_blocks.append({"type": "text", "text": "[Video]"}) # 2. 额外的内容块(系统提醒、指令等) if self.extra_user_content_parts: @@ -245,6 +261,40 @@ async def assemble_context(self) -> dict: "url": resolved.to_data_url(), }, } + # A video supplied through `extra_user_content_parts` is handled + # like `video_urls` below: resolve it to a portable data URI and + # flag it transient. Persisting it would replay the whole payload + # on every later request, and a local path may be gone by then. + if isinstance(dumped, dict) and dumped.get("type") == "video_url": + video_url = dumped.get("video_url") + url = video_url.get("url") if isinstance(video_url, dict) else None + if isinstance(url, str) and url: + try: + resolved_video = await MediaResolver( + url, + media_type="video", + default_suffix=".mp4", + ).to_base64_data(strict=True) + except Exception as exc: + logger.warning( + "Video preprocessing failed; skipping it. Error: %s", + exc, + ) + continue + if not resolved_video: + logger.warning( + "Video preprocessing returned no data; skipping it." + ) + continue + dumped = { + **dumped, + "video_url": { + **video_url, + "url": resolved_video.to_data_url(), + }, + "_no_save": True, + } + transient_video_added = True content_blocks.append(dumped) # 3. Read image references without resizing or transcoding. @@ -299,12 +349,49 @@ async def assemble_context(self) -> dict: }, ) + # 5. Video content. The block is flagged `_no_save` so that the + # multi-megabyte data URI is dropped when the message is written back to + # the conversation history; otherwise it would be replayed on every later + # request in the session. Providers still receive it, because they + # serialize messages with `model_dump()`, which does not emit `_no_save`. + if self.video_urls: + for video_url in self.video_urls: + try: + video_data = await MediaResolver( + video_url, + media_type="video", + default_suffix=".mp4", + ).to_base64_data(strict=True) + except Exception as exc: + logger.warning( + "Video preprocessing failed; skipping it. Error: %s", exc + ) + continue + if not video_data: + logger.warning("Video preprocessing returned no data; skipping it.") + continue + content_blocks.append( + { + "type": "video_url", + "video_url": {"url": video_data.to_data_url()}, + "_no_save": True, + }, + ) + if image_capture_failed and not any( block.get("type") != "text" or block.get("text", "").strip() for block in content_blocks ): content_blocks = [{"type": "text", "text": "[Image unavailable]"}] + # A message made only of transient blocks would be persisted with an + # empty content list and replayed that way on every later request, so + # keep a text placeholder for the attachment. + if transient_video_added and not any( + not block.get("_no_save") for block in content_blocks + ): + content_blocks.insert(0, {"type": "text", "text": "[Video]"}) + # 只有当只有一个来自 prompt 的文本块且没有额外内容块时,才降级为简单格式以保持向后兼容 if ( len(content_blocks) == 1 @@ -312,6 +399,7 @@ async def assemble_context(self) -> dict: and not self.extra_user_content_parts and not self.image_urls and not self.audio_urls + and not self.video_urls ): return {"role": "user", "content": content_blocks[0]["text"]} diff --git a/astrbot/core/provider/modalities.py b/astrbot/core/provider/modalities.py index 66ac74e9b72..b028d7541bd 100644 --- a/astrbot/core/provider/modalities.py +++ b/astrbot/core/provider/modalities.py @@ -13,6 +13,7 @@ class ContextSanitizeStats: fixed_image_blocks: int = 0 fixed_audio_blocks: int = 0 + fixed_video_blocks: int = 0 fixed_tool_messages: int = 0 removed_tool_calls: int = 0 @@ -21,6 +22,7 @@ def changed(self) -> bool: return bool( self.fixed_image_blocks or self.fixed_audio_blocks + or self.fixed_video_blocks or self.fixed_tool_messages or self.removed_tool_calls ) @@ -34,6 +36,33 @@ def _message_to_dict(message: dict[str, Any] | Message) -> dict[str, Any] | None return None +def has_video_blocks(contexts: Sequence[dict[str, Any] | Message]) -> bool: + """Whether any message in `contexts` carries a `video_url` block. + + Accepts both `Message` objects (whose content holds `ContentPart` instances) + and plain dicts, since callers pass either form. + + Args: + contexts: Messages to inspect. + + Returns: + True as soon as a `video_url` block is found. + """ + for msg in contexts: + content = msg.content if isinstance(msg, Message) else msg.get("content") + if not isinstance(content, list): + continue + for part in content: + part_type = ( + part.get("type") + if isinstance(part, dict) + else getattr(part, "type", None) + ) + if part_type == "video_url": + return True + return False + + def sanitize_contexts_by_modalities( contexts: Sequence[dict[str, Any] | Message], modalities: list[str] | None, @@ -41,17 +70,43 @@ def sanitize_contexts_by_modalities( if not contexts: return [], ContextSanitizeStats() if not modalities or not isinstance(modalities, list): + # An unconfigured provider (None or an empty list) keeps the historical + # "supports everything" default for image/audio/tool_use. Video postdates + # that convention and stays strictly opt-in, so it is degraded even here: + # this function also sanitizes a context that was assembled for another + # provider, which may have produced a `video_url` block because *it* + # declared the video modality. Everything else is copied through + # untouched, exactly as before. copied_contexts = [] + stats = ContextSanitizeStats() for msg in contexts: copied_msg = _message_to_dict(msg) - if copied_msg: - copied_contexts.append(copied_msg) - return copied_contexts, ContextSanitizeStats() + if not copied_msg: + continue + content = copied_msg.get("content") + if isinstance(content, list): + filtered_parts: list[Any] = [] + removed_video = False + for part in content: + if ( + isinstance(part, dict) + and str(part.get("type", "")).lower() == "video_url" + ): + removed_video = True + stats.fixed_video_blocks += 1 + filtered_parts.append({"type": "text", "text": "[Video]"}) + continue + filtered_parts.append(part) + if removed_video: + copied_msg["content"] = filtered_parts + copied_contexts.append(copied_msg) + return copied_contexts, stats supports_image = "image" in modalities supports_audio = "audio" in modalities + supports_video = "video" in modalities supports_tool_use = "tool_use" in modalities - if supports_image and supports_audio and supports_tool_use: + if supports_image and supports_audio and supports_video and supports_tool_use: copied_contexts = [] for msg in contexts: copied_msg = _message_to_dict(msg) @@ -83,7 +138,7 @@ def sanitize_contexts_by_modalities( msg.pop("tool_calls", None) msg.pop("tool_call_id", None) - if not supports_image or not supports_audio: + if not supports_image or not supports_audio or not supports_video: content = msg.get("content") if isinstance(content, list): filtered_parts: list[Any] = [] @@ -104,6 +159,11 @@ def sanitize_contexts_by_modalities( stats.fixed_audio_blocks += 1 filtered_parts.append({"type": "text", "text": "[Audio]"}) continue + if not supports_video and part_type == "video_url": + removed_any_multimodal = True + stats.fixed_video_blocks += 1 + filtered_parts.append({"type": "text", "text": "[Video]"}) + continue filtered_parts.append(part) if removed_any_multimodal: msg["content"] = filtered_parts @@ -136,6 +196,8 @@ def _tool_result_placeholder(content: Any) -> str: text_parts.append("[Image]") elif part_type in {"audio_url", "input_audio"}: text_parts.append("[Audio]") + elif part_type == "video_url": + text_parts.append("[Video]") content_text = "\n".join(part for part in text_parts if part).strip() else: content_text = "" @@ -149,10 +211,11 @@ def log_context_sanitize_stats(stats: ContextSanitizeStats) -> None: return logger.debug( "context modality fix applied: " - "fixed_image_blocks=%s, fixed_audio_blocks=%s, " + "fixed_image_blocks=%s, fixed_audio_blocks=%s, fixed_video_blocks=%s, " "fixed_tool_messages=%s, removed_tool_calls=%s", stats.fixed_image_blocks, stats.fixed_audio_blocks, + stats.fixed_video_blocks, stats.fixed_tool_messages, stats.removed_tool_calls, ) diff --git a/astrbot/core/provider/sources/anthropic_source.py b/astrbot/core/provider/sources/anthropic_source.py index 20a3a813d5d..a07ce575133 100644 --- a/astrbot/core/provider/sources/anthropic_source.py +++ b/astrbot/core/provider/sources/anthropic_source.py @@ -13,7 +13,13 @@ from astrbot import logger from astrbot.api.provider import Provider -from astrbot.core.agent.message import AudioURLPart, ContentPart, ImageURLPart, TextPart +from astrbot.core.agent.message import ( + AudioURLPart, + ContentPart, + ImageURLPart, + TextPart, + VideoURLPart, +) from astrbot.core.exceptions import EmptyModelOutputError from astrbot.core.provider.entities import LLMResponse, TokenUsage from astrbot.core.provider.func_tool_manager import ToolSet @@ -1023,6 +1029,10 @@ async def resolve_image_url(image_url: str) -> dict | None: content.append(image_dict) elif isinstance(block, AudioURLPart): content.append({"type": "text", "text": "[Audio]"}) + elif isinstance(block, VideoURLPart): + # No native video input here and history is replayed, so + # degrade to a text placeholder instead of resolving base64. + content.append({"type": "text", "text": "[Video]"}) else: raise ValueError(f"不支持的额外内容块类型: {type(block)}") diff --git a/astrbot/core/provider/sources/gemini_source.py b/astrbot/core/provider/sources/gemini_source.py index 0d8df6af99b..63151aff74a 100644 --- a/astrbot/core/provider/sources/gemini_source.py +++ b/astrbot/core/provider/sources/gemini_source.py @@ -15,7 +15,13 @@ import astrbot.core.message.components as Comp from astrbot import logger from astrbot.api.provider import Provider -from astrbot.core.agent.message import AudioURLPart, ContentPart, ImageURLPart, TextPart +from astrbot.core.agent.message import ( + AudioURLPart, + ContentPart, + ImageURLPart, + TextPart, + VideoURLPart, +) from astrbot.core.exceptions import EmptyModelOutputError from astrbot.core.message.message_event_result import MessageChain from astrbot.core.provider.entities import LLMResponse, TokenUsage @@ -407,6 +413,22 @@ def process_audio_url(audio_url_dict: dict) -> types.Part: audio_bytes = base64.b64decode(url.split(",", 1)[1]) return types.Part.from_bytes(data=audio_bytes, mime_type=mime_type) + async def process_video_url(video_url_dict: dict) -> types.Part: + url = video_url_dict["url"] + video_data = await resolve_media_ref_to_base64_data( + url, + media_type="video", + strict=True, + ) + if video_data is None: + raise ValueError( + f"Failed to resolve Gemini history video: {describe_media_ref(url)}" + ) + return types.Part.from_bytes( + data=base64.b64decode(video_data.base64_data), + mime_type=video_data.mime_type, + ) + def append_or_extend( contents: list[types.Content], part: list[types.Part], @@ -430,8 +452,16 @@ def append_or_extend( parts.append(types.Part.from_text(text=item["text"] or " ")) elif item["type"] == "image_url": parts.append(await process_image_url(item["image_url"])) - else: + elif item["type"] == "audio_url": parts.append(process_audio_url(item["audio_url"])) + elif item["type"] == "video_url": + parts.append(await process_video_url(item["video_url"])) + else: + # Never raise for an unknown block type here. This + # message is replayed from the session history on + # every later request, so an exception would keep + # failing long after the original turn. + parts.append(create_text_part(f"[{item['type']}]")) else: parts = [create_text_part(content)] append_or_extend(gemini_contents, parts, types.UserContent) @@ -944,6 +974,7 @@ async def text_chat( session_id=None, image_urls=None, audio_urls=None, + video_urls=None, func_tool=None, contexts=None, system_prompt=None, @@ -963,6 +994,7 @@ async def text_chat( prompt or "", image_urls, audio_urls, + video_urls, extra_user_content_parts, ) context_query = self._ensure_message_to_dicts(contexts) @@ -1016,6 +1048,7 @@ async def text_chat_stream( session_id=None, image_urls=None, audio_urls=None, + video_urls=None, func_tool=None, contexts=None, system_prompt=None, @@ -1035,6 +1068,7 @@ async def text_chat_stream( prompt or "", image_urls, audio_urls, + video_urls, extra_user_content_parts, ) context_query = self._ensure_message_to_dicts(contexts) @@ -1113,6 +1147,7 @@ async def assemble_context( text: str, image_urls: list[str] | None = None, audio_urls: list[str] | None = None, + video_urls: list[str] | None = None, extra_user_content_parts: list[ContentPart] | None = None, ): """组装上下文。""" @@ -1151,6 +1186,27 @@ async def resolve_audio_part(audio_path: str) -> dict | None: "audio_url": {"url": audio_data.to_data_url()}, } + async def resolve_video_part(video_ref: str) -> dict | None: + try: + video_data = await resolve_media_ref_to_base64_data( + video_ref, + media_type="video", + strict=True, + ) + except Exception as exc: + logger.warning( + "Video preprocessing failed; ignoring it. Error: %s", exc + ) + return None + + if not video_data: + logger.warning("Video preprocessing returned no data; ignoring it.") + return None + return { + "type": "video_url", + "video_url": {"url": video_data.to_data_url()}, + } + # 构建内容块列表 content_blocks = [] @@ -1162,6 +1218,9 @@ async def resolve_audio_part(audio_path: str) -> dict | None: content_blocks.append({"type": "text", "text": "[Image]"}) elif audio_urls: content_blocks.append({"type": "text", "text": "[Audio]"}) + elif video_urls: + # If there is no text but there is video, add a placeholder text. + content_blocks.append({"type": "text", "text": "[Video]"}) elif extra_user_content_parts: # 如果只有额外内容块,也需要添加占位文本 content_blocks.append({"type": "text", "text": " "}) @@ -1179,6 +1238,10 @@ async def resolve_audio_part(audio_path: str) -> dict | None: audio_part = await resolve_audio_part(part.audio_url.url) if audio_part: content_blocks.append(audio_part) + elif isinstance(part, VideoURLPart): + video_part = await resolve_video_part(part.video_url.url) + if video_part: + content_blocks.append(video_part) else: raise ValueError( f"Unsupported extra content part type: {type(part)}" @@ -1197,12 +1260,19 @@ async def resolve_audio_part(audio_path: str) -> dict | None: if audio_part: content_blocks.append(audio_part) + if video_urls: + for video_ref in video_urls: + video_part = await resolve_video_part(video_ref) + if video_part: + content_blocks.append(video_part) + # 如果只有主文本且没有额外内容块和图片,返回简单格式以保持向后兼容 if ( text and not extra_user_content_parts and not image_urls and not audio_urls + and not video_urls and len(content_blocks) == 1 and content_blocks[0]["type"] == "text" ): diff --git a/astrbot/core/provider/sources/openai_source.py b/astrbot/core/provider/sources/openai_source.py index 9b0c89d5d28..f4f7534d008 100644 --- a/astrbot/core/provider/sources/openai_source.py +++ b/astrbot/core/provider/sources/openai_source.py @@ -24,6 +24,7 @@ ImageURLPart, Message, TextPart, + VideoURLPart, ) from astrbot.core.agent.tool import ToolSet from astrbot.core.exceptions import EmptyModelOutputError @@ -1420,6 +1421,10 @@ async def assemble_context( audio_part = await self._resolve_audio_part(part.audio_url.url) if audio_part: content_blocks.append(audio_part) + elif isinstance(part, VideoURLPart): + # No native video input here and history is replayed, so + # degrade to a text placeholder instead of resolving base64. + content_blocks.append({"type": "text", "text": "[Video]"}) else: raise ValueError(f"不支持的额外内容块类型: {type(part)}") diff --git a/dashboard/src/components/provider/ProviderChatCompletionPanel.vue b/dashboard/src/components/provider/ProviderChatCompletionPanel.vue index 2500782e611..a6c334db80f 100644 --- a/dashboard/src/components/provider/ProviderChatCompletionPanel.vue +++ b/dashboard/src/components/provider/ProviderChatCompletionPanel.vue @@ -98,6 +98,7 @@ :is-source-modified="isSourceModified" :supports-image-input="supportsImageInput" :supports-audio-input="supportsAudioInput" + :supports-video-input="supportsVideoInput" :supports-tool-call="supportsToolCall" :supports-reasoning="supportsReasoning" :format-context-limit="formatContextLimit" @@ -248,6 +249,7 @@ const { getSourceDisplayName, supportsImageInput, supportsAudioInput, + supportsVideoInput, supportsToolCall, supportsReasoning, formatContextLimit, diff --git a/dashboard/src/components/provider/ProviderModelsPanel.vue b/dashboard/src/components/provider/ProviderModelsPanel.vue index 9826e1cb7ea..70ebead4451 100644 --- a/dashboard/src/components/provider/ProviderModelsPanel.vue +++ b/dashboard/src/components/provider/ProviderModelsPanel.vue @@ -296,6 +296,10 @@ const props = defineProps({ type: Function, required: true }, + supportsVideoInput: { + type: Function, + required: true + }, supportsToolCall: { type: Function, required: true @@ -367,6 +371,15 @@ const capabilityBadges = (entry) => { enabled: !isConfigured || modalities.includes('audio'), label: props.tm('models.metadata.audio') }, + { + key: 'video', + icon: 'mdi-video-outline', + supported: props.supportsVideoInput(metadata), + // Video is opt-in on the backend, so an unconfigured provider must not + // report it as enabled the way image/audio do. + enabled: modalities.includes('video'), + label: props.tm('models.metadata.video') + }, { key: 'tool_use', icon: 'mdi-wrench-outline', diff --git a/dashboard/src/composables/useProviderSources.ts b/dashboard/src/composables/useProviderSources.ts index aea46f37083..b53e9d5d9ef 100644 --- a/dashboard/src/composables/useProviderSources.ts +++ b/dashboard/src/composables/useProviderSources.ts @@ -365,6 +365,11 @@ export function useProviderSources(options: UseProviderSourcesOptions) { return inputs.includes('audio') } + function supportsVideoInput(meta: any) { + const inputs = meta?.modalities?.input || [] + return inputs.includes('video') + } + function supportsToolCall(meta: any) { return Boolean(meta?.tool_call) } @@ -648,6 +653,11 @@ export function useProviderSources(options: UseProviderSourcesOptions) { if (supportsAudioInput(metadata)) { modalities.push('audio') } + // Video is opt-in: only enable it when the model metadata declares video + // input, never as part of the "assume everything" fallback above. + if (supportsVideoInput(metadata)) { + modalities.push('video') + } if (supportsToolCall(metadata)) { modalities.push('tool_use') } @@ -831,6 +841,7 @@ export function useProviderSources(options: UseProviderSourcesOptions) { getModelMetadata, supportsImageInput, supportsAudioInput, + supportsVideoInput, supportsToolCall, supportsReasoning, formatContextLimit, diff --git a/dashboard/src/i18n/locales/en-US/features/provider.json b/dashboard/src/i18n/locales/en-US/features/provider.json index aad194fcd6f..5bd1cfc5f9b 100644 --- a/dashboard/src/i18n/locales/en-US/features/provider.json +++ b/dashboard/src/i18n/locales/en-US/features/provider.json @@ -165,6 +165,7 @@ "metadata": { "image": "Image input", "audio": "Audio input", + "video": "Video input", "toolUse": "Tool use", "reasoning": "Reasoning", "context": "{tokens} context window", diff --git a/dashboard/src/i18n/locales/ja-JP/features/provider.json b/dashboard/src/i18n/locales/ja-JP/features/provider.json index ef95b6b1e1a..0086bf16b55 100644 --- a/dashboard/src/i18n/locales/ja-JP/features/provider.json +++ b/dashboard/src/i18n/locales/ja-JP/features/provider.json @@ -165,6 +165,7 @@ "metadata": { "image": "画像入力", "audio": "音声入力", + "video": "動画入力", "toolUse": "ツール使用", "reasoning": "推論", "context": "{tokens} トークンのコンテキストウィンドウ", diff --git a/dashboard/src/i18n/locales/ru-RU/features/provider.json b/dashboard/src/i18n/locales/ru-RU/features/provider.json index 3a8dbc7bbe8..8d9282dfab4 100644 --- a/dashboard/src/i18n/locales/ru-RU/features/provider.json +++ b/dashboard/src/i18n/locales/ru-RU/features/provider.json @@ -165,6 +165,7 @@ "metadata": { "image": "Ввод изображений", "audio": "Ввод аудио", + "video": "Ввод видео", "toolUse": "Использование инструментов", "reasoning": "Рассуждение", "context": "Контекстное окно {tokens}", diff --git a/dashboard/src/i18n/locales/zh-CN/features/provider.json b/dashboard/src/i18n/locales/zh-CN/features/provider.json index 0818e442d9c..84d69cb6648 100644 --- a/dashboard/src/i18n/locales/zh-CN/features/provider.json +++ b/dashboard/src/i18n/locales/zh-CN/features/provider.json @@ -165,6 +165,7 @@ "metadata": { "image": "图像输入", "audio": "音频输入", + "video": "视频输入", "toolUse": "工具使用", "reasoning": "推理", "context": "{tokens} 上下文窗口", diff --git a/dashboard/src/utils/providerMetadata.ts b/dashboard/src/utils/providerMetadata.ts index e5fe7e2e51b..9e37a135354 100644 --- a/dashboard/src/utils/providerMetadata.ts +++ b/dashboard/src/utils/providerMetadata.ts @@ -65,6 +65,13 @@ export function providerCapabilityBadges( enabled: modalities.includes('audio'), label: tm('models.metadata.audio') }, + { + key: 'video', + icon: 'mdi-video-outline', + supported: inputs.includes('video'), + enabled: modalities.includes('video'), + label: tm('models.metadata.video') + }, { key: 'tool_use', icon: 'mdi-wrench-outline', diff --git a/dashboard/src/views/ProviderPage.vue b/dashboard/src/views/ProviderPage.vue index 5065bc8afdf..c2f92796600 100644 --- a/dashboard/src/views/ProviderPage.vue +++ b/dashboard/src/views/ProviderPage.vue @@ -138,6 +138,7 @@ :is-source-modified="isSourceModified" :supports-image-input="supportsImageInput" :supports-audio-input="supportsAudioInput" + :supports-video-input="supportsVideoInput" :supports-tool-call="supportsToolCall" :supports-reasoning="supportsReasoning" :format-context-limit="formatContextLimit" @@ -364,6 +365,7 @@ const { getSourceDisplayName, supportsImageInput, supportsAudioInput, + supportsVideoInput, supportsToolCall, supportsReasoning, formatContextLimit, diff --git a/tests/test_video_input_modality.py b/tests/test_video_input_modality.py new file mode 100644 index 00000000000..21cffa93136 --- /dev/null +++ b/tests/test_video_input_modality.py @@ -0,0 +1,401 @@ +from types import SimpleNamespace + +import pytest + +from astrbot.core.agent.message import ( + Message, + TextPart, + VideoURLPart, + dump_messages_with_checkpoints, +) +from astrbot.core.agent.runners.tool_loop_agent_runner import ToolLoopAgentRunner +from astrbot.core.provider.entities import ProviderRequest +from astrbot.core.provider.modalities import sanitize_contexts_by_modalities +from astrbot.core.provider.sources.gemini_source import ProviderGoogleGenAI + +VIDEO_DATA_URL = "data:video/mp4;base64,AAAAGGZ0eXBtcDQyAAAAAG1wNDJpc29t" + + +def _gemini_provider() -> ProviderGoogleGenAI: + provider = ProviderGoogleGenAI.__new__(ProviderGoogleGenAI) + provider.provider_config = {} + provider.provider_settings = {} + return provider + + +def _runner(modalities) -> ToolLoopAgentRunner: + runner = ToolLoopAgentRunner.__new__(ToolLoopAgentRunner) + runner.provider = SimpleNamespace(provider_config={"modalities": modalities}) + return runner + + +def test_video_url_part_is_registered_and_round_trips(): + """A `video_url` dict must resolve through the ContentPart registry. + + The registry lookup in `ContentPart.__get_pydantic_core_schema__` is + unguarded, so an unregistered type raises `KeyError` the moment the block is + validated from persisted history. + """ + message = Message.model_validate( + { + "role": "user", + "content": [{"type": "video_url", "video_url": {"url": VIDEO_DATA_URL}}], + } + ) + + assert isinstance(message.content[0], VideoURLPart) + assert message.content[0].video_url.url == VIDEO_DATA_URL + + +@pytest.mark.asyncio +async def test_assemble_context_marks_video_block_as_transient(): + request = ProviderRequest(prompt="", video_urls=[VIDEO_DATA_URL]) + + context = await request.assemble_context() + + video_blocks = [ + block for block in context["content"] if block.get("type") == "video_url" + ] + assert len(video_blocks) == 1 + assert video_blocks[0]["_no_save"] is True + + +@pytest.mark.asyncio +async def test_video_block_is_not_written_to_history(): + """The video payload must not be persisted and replayed on later requests.""" + request = ProviderRequest(prompt="look", video_urls=[VIDEO_DATA_URL]) + context = await request.assemble_context() + + messages = [Message.model_validate(context)] + dumped = dump_messages_with_checkpoints(messages) + + persisted_types = [ + part["type"] for part in dumped[0]["content"] if isinstance(part, dict) + ] + assert "video_url" not in persisted_types + assert "text" in persisted_types + + +@pytest.mark.asyncio +async def test_transient_video_reaches_provider_but_not_history(): + """Prove both halves of the design in one place. + + Providers serialize messages with `model_dump()`, which does not emit + `_no_save`, so the video still reaches the provider payload. Persistence goes + through `dump_messages_with_checkpoints`, which drops it. + """ + request = ProviderRequest(prompt="look", video_urls=[VIDEO_DATA_URL]) + context = await request.assemble_context() + message = Message.model_validate(context) + + assert message.content[1]._no_save is True + + provider_payload = message.model_dump() + assert any(part["type"] == "video_url" for part in provider_payload["content"]) + + persisted = dump_messages_with_checkpoints([message]) + assert not any(part["type"] == "video_url" for part in persisted[0]["content"]) + + +@pytest.mark.asyncio +async def test_gemini_accepts_video_url_block(): + provider = _gemini_provider() + payloads = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe this"}, + {"type": "video_url", "video_url": {"url": VIDEO_DATA_URL}}, + ], + } + ] + } + + contents = await provider._prepare_conversation(payloads) + + parts = contents[0].parts + assert len(parts) == 2 + assert parts[1].inline_data is not None + assert parts[1].inline_data.mime_type == "video/mp4" + + +@pytest.mark.asyncio +async def test_gemini_degrades_unknown_block_type_instead_of_raising(): + """An unknown block must not raise. + + The message is replayed from history on every later request, so raising here + would keep failing long after the turn that produced it. + """ + provider = _gemini_provider() + payloads = { + "messages": [ + { + "role": "user", + "content": [{"type": "some_future_block", "payload": "x"}], + } + ] + } + + contents = await provider._prepare_conversation(payloads) + + assert contents[0].parts[0].text == "[some_future_block]" + + +@pytest.mark.asyncio +async def test_video_is_opt_in_for_providers_without_modalities(): + """An unconfigured provider must not receive video. + + An empty or missing `modalities` list means "supports everything" for + backward compatibility, but video postdates that convention and is not + universally supported. + """ + runner = _runner([]) + request = ProviderRequest(prompt="look", video_urls=[VIDEO_DATA_URL]) + + context = await runner._assemble_request_context_for_provider(request) + + # With the video removed this collapses to the plain-text form, which is the + # pre-existing behaviour for a text-only request. + content = context["content"] + if isinstance(content, str): + assert "video_url" not in content + else: + assert not any(block.get("type") == "video_url" for block in content) + + +@pytest.mark.asyncio +async def test_video_is_sent_when_declared(): + runner = _runner(["text", "image", "audio", "video", "tool_use"]) + request = ProviderRequest(prompt="look", video_urls=[VIDEO_DATA_URL]) + + context = await runner._assemble_request_context_for_provider(request) + + assert any(block.get("type") == "video_url" for block in context["content"]) + + +@pytest.mark.asyncio +async def test_video_is_replaced_with_placeholder_when_not_declared(): + runner = _runner(["text", "image", "audio", "tool_use"]) + request = ProviderRequest(prompt="look", video_urls=[VIDEO_DATA_URL]) + + context = await runner._assemble_request_context_for_provider(request) + + content = context["content"] + assert not any(block.get("type") == "video_url" for block in content) + assert {"type": "text", "text": "[Video]"} in content + + +def test_modality_sanitizer_degrades_video_to_text(): + contexts = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "video_url", "video_url": {"url": VIDEO_DATA_URL}}, + ], + } + ] + + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text", "image"]) + + assert stats.fixed_video_blocks == 1 + assert {"type": "text", "text": "[Video]"} in sanitized[0]["content"] + + +def test_modality_sanitizer_keeps_video_when_declared(): + contexts = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "video_url", "video_url": {"url": VIDEO_DATA_URL}}, + ], + } + ] + + sanitized, stats = sanitize_contexts_by_modalities( + contexts, ["text", "image", "audio", "video"] + ) + + assert stats.fixed_video_blocks == 0 + assert any(block.get("type") == "video_url" for block in sanitized[0]["content"]) + + +@pytest.mark.asyncio +async def test_video_placeholder_is_added_for_video_only_requests(): + request = ProviderRequest(prompt="", video_urls=[VIDEO_DATA_URL]) + + context = await request.assemble_context() + + assert context["content"][0] == {"type": "text", "text": "[Video]"} + + +def test_text_only_request_is_unchanged(): + """The simple string form must be preserved for text-only requests.""" + request = ProviderRequest(prompt="hello") + + assert request.video_urls == [] + assert TextPart(text="hello").text == "hello" + + +def _video_context() -> list[dict]: + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "video_url", "video_url": {"url": VIDEO_DATA_URL}}, + ], + } + ] + + +@pytest.mark.parametrize("modalities", [[], None]) +def test_unconfigured_sanitizer_degrades_video(modalities): + """Video must stay opt-in on the sanitize path too. + + `sanitize_contexts_by_modalities` also runs on an already-assembled context + when a request falls back to another provider. That context may carry a + `video_url` block produced by the primary provider, so an unconfigured + fallback must not inherit it just because empty modalities means "supports + everything" for the other modalities. + """ + sanitized, stats = sanitize_contexts_by_modalities(_video_context(), modalities) + + assert stats.fixed_video_blocks == 1 + assert stats.changed is True + assert not any(part.get("type") == "video_url" for part in sanitized[0]["content"]) + assert {"type": "text", "text": "[Video]"} in sanitized[0]["content"] + + +def test_unconfigured_sanitizer_leaves_the_rest_of_the_history_alone(): + """Only video is degraded for an unconfigured provider. + + Everything else must be copied through untouched, including messages that + the configured-provider path would filter out. + """ + contexts = [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": ""}, + {"content": "message without a role"}, + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://x/i.png"}}], + }, + ] + + sanitized, stats = sanitize_contexts_by_modalities(contexts, []) + + assert sanitized == contexts + assert stats.changed is False + + +def test_fallback_sanitizer_strips_video_for_unconfigured_provider(): + """The runner's fallback path must apply the same opt-in rule. + + The primary provider may have declared video while the fallback provider is + unconfigured, and the context is sanitized only once the fallback is tried. + """ + runner = _runner([]) + + sanitized = runner._sanitize_contexts_for_provider(_video_context()) + + content = sanitized[0]["content"] + assert not any(part.get("type") == "video_url" for part in content) + assert {"type": "text", "text": "[Video]"} in content + + +def test_unconfigured_provider_keeps_its_context_objects_without_video(): + """The sanitizer is only entered when there is a video block to degrade. + + An unconfigured provider must keep receiving the very same context objects + it always did; converting a plain-text history to dicts is an unrelated + behaviour change. + """ + runner = _runner([]) + contexts = [{"role": "user", "content": "hello"}] + + assert runner._sanitize_contexts_for_provider(contexts) is contexts + + +def test_fallback_sanitizer_strips_video_from_message_objects(): + """The production path hands the sanitizer `Message` objects, not dicts.""" + runner = _runner([]) + message = Message.model_validate(_video_context()[0]) + + sanitized = runner._sanitize_contexts_for_provider([message]) + + content = sanitized[0]["content"] + assert not any(part.get("type") == "video_url" for part in content) + assert {"type": "text", "text": "[Video]"} in content + + +@pytest.mark.asyncio +async def test_extra_video_part_is_resolved_and_marked_transient(): + """A `VideoURLPart` in `extra_user_content_parts` must behave like `video_urls`. + + It has to be resolved to a portable data URI and flagged transient, or the + payload is persisted and a local path is replayed after the file is gone. + """ + request = ProviderRequest( + prompt="look", + extra_user_content_parts=[ + VideoURLPart(video_url=VideoURLPart.VideoURL(url=VIDEO_DATA_URL)) + ], + ) + + context = await request.assemble_context() + + video_blocks = [b for b in context["content"] if b.get("type") == "video_url"] + assert len(video_blocks) == 1 + assert video_blocks[0]["_no_save"] is True + assert video_blocks[0]["video_url"]["url"].startswith("data:video/mp4;base64,") + + +@pytest.mark.asyncio +async def test_extra_video_part_reaches_provider_but_not_history(): + request = ProviderRequest( + prompt="look", + extra_user_content_parts=[ + VideoURLPart(video_url=VideoURLPart.VideoURL(url=VIDEO_DATA_URL)) + ], + ) + message = Message.model_validate(await request.assemble_context()) + + assert any(part["type"] == "video_url" for part in message.model_dump()["content"]) + + persisted = dump_messages_with_checkpoints([message]) + assert not any(part["type"] == "video_url" for part in persisted[0]["content"]) + + +@pytest.mark.asyncio +async def test_extra_video_only_message_keeps_a_text_placeholder(): + """A message of only transient blocks must not persist as empty content.""" + request = ProviderRequest( + prompt="", + extra_user_content_parts=[ + VideoURLPart(video_url=VideoURLPart.VideoURL(url=VIDEO_DATA_URL)) + ], + ) + message = Message.model_validate(await request.assemble_context()) + + persisted = dump_messages_with_checkpoints([message]) + assert not any(part.get("type") == "video_url" for part in persisted[0]["content"]) + assert {"type": "text", "text": "[Video]"} in persisted[0]["content"] + + +@pytest.mark.asyncio +async def test_extra_video_part_is_dropped_when_it_cannot_be_resolved(tmp_path): + request = ProviderRequest( + prompt="look", + extra_user_content_parts=[ + VideoURLPart( + video_url=VideoURLPart.VideoURL(url=str(tmp_path / "missing.mp4")) + ) + ], + ) + + context = await request.assemble_context() + + assert not any(b.get("type") == "video_url" for b in context["content"])