diff --git a/.github/workflows/ubuntu-tests.yml b/.github/workflows/ubuntu-tests.yml index 2870e8a9d5d..8449069cdca 100644 --- a/.github/workflows/ubuntu-tests.yml +++ b/.github/workflows/ubuntu-tests.yml @@ -54,8 +54,7 @@ jobs: pytest-mock \ -r requirements/requirements.in \ -r requirements/requirements-help.in \ - -r requirements/requirements-playwright.in \ - ".[help,playwright]" + ".[help]" env: PYO3_USE_ABI3_FORWARD_COMPATIBILITY: "1" diff --git a/.github/workflows/windows-tests.yml b/.github/workflows/windows-tests.yml index 924f21f4cf3..e5e3ea1f02a 100644 --- a/.github/workflows/windows-tests.yml +++ b/.github/workflows/windows-tests.yml @@ -42,7 +42,7 @@ jobs: run: | python -m pip install --upgrade pip pip install uv - uv pip install --system pytest pytest-asyncio pytest-mock -r requirements/requirements.in -r requirements/requirements-help.in -r requirements/requirements-playwright.in '.[help,playwright]' + uv pip install --system pytest pytest-asyncio pytest-mock -r requirements/requirements.in -r requirements/requirements-help.in '.[help]' env: PYO3_USE_ABI3_FORWARD_COMPATIBILITY: "1" diff --git a/MANIFEST.in b/MANIFEST.in index 09cfe029620..a848a0f0b41 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,7 +3,6 @@ include requirements/requirements.in include requirements/requirements-dev.in include requirements/requirements-help.in -include requirements/requirements-playwright.in global-exclude .DS_Store diff --git a/cecli/args.py b/cecli/args.py index 9953a734dae..1857db4eaf7 100644 --- a/cecli/args.py +++ b/cecli/args.py @@ -56,6 +56,7 @@ DEEP_MERGE_JSON_FIELDS: frozenset[str] = frozenset( { "agent_config", + "browser", "mcp_servers", "hooks", "model_providers", @@ -547,6 +548,17 @@ def get_parser(default_config_files, git_root): ) ########## + group = parser.add_argument_group("Browser Settings") + group.add_argument( + "--browser", + metavar="BROWSER_CONFIG_JSON", + default=None, + help=( + "Specify browser-harness configuration as a JSON/YAML string or file path (e.g., " + '\'{"cdp_url": "http://127.0.0.1:9222", "launch": true}\')' + ), + ) + group = parser.add_argument_group("Security Settings") group.add_argument( "--security-config", @@ -1145,12 +1157,6 @@ def get_parser(default_config_files, git_root): help="Always say yes to every confirmation (including cli commands)", default=False, ) - group.add_argument( - "--disable-playwright", - action="store_true", - help="Never prompt for or attempt to install Playwright for web scraping (default: False).", - default=False, - ) group.add_argument( "--disable-scraping", action="store_true", diff --git a/cecli/coders/agent_coder.py b/cecli/coders/agent_coder.py index 184080e2046..14977af8fe6 100644 --- a/cecli/coders/agent_coder.py +++ b/cecli/coders/agent_coder.py @@ -19,6 +19,7 @@ from cecli.helpers.agents.service import AgentService from cecli.helpers.agents.tree import AgentTree from cecli.helpers.background_commands import BackgroundCommandManager +from cecli.helpers.browser import export_active_cdp_url from cecli.helpers.conversation import ConversationService, MessageTag from cecli.helpers.similarity import ( cosine_similarity, @@ -131,6 +132,8 @@ def __init__(self, *args, **kwargs): def post_init(self): super().post_init() + export_active_cdp_url(getattr(self.args, "browser", None), include_default_ports=True) + if not self._inherited_tools: # Populate per-instance tool and server filters from config self.registered_tools["included"] = set( @@ -265,6 +268,7 @@ def _get_agent_config(self, config=None): # "context_summary", # "directory_structure", "environment_info", + "browser_automation", # "git_status", # "symbol_outline", "todo_list", @@ -496,6 +500,7 @@ def _calculate_context_block_tokens(self, force=False): self.context_blocks_cache = {} block_types = [ "environment_info", + "browser_automation", "directory_structure", "git_status", "symbol_outline", @@ -538,6 +543,8 @@ def _generate_context_block(self, block_name): content = self.get_todo_list() elif block_name == "skills": content = self.get_skills_context() + elif block_name == "browser_automation": + content = self.get_browser_automation_context() elif block_name == "servers": content = self.get_servers_context() elif block_name == "orchestration": @@ -1711,6 +1718,32 @@ def get_skills_content(self): self.io.tool_error(f"Error generating skills content context: {str(e)}") return None + def get_browser_automation_context(self): + """ + Generate a brief browser-harness usage context block. + + Returns None when browser-harness is not installed. + """ + from cecli.helpers.browser import is_browser_harness_available + + if not self.use_enhanced_context or not is_browser_harness_available(): + return None + + result = '\n' + result += "## Browser Automation\n\n" + result += ( + "`browser-harness` drives a real Chrome/Chromium browser. Run it with a heredoc;" + " cecli sets `BU_CDP_URL` automatically so it attaches to a CDP browser:\n\n" + "```bash\n" + "browser-harness <<'PY'\n" + "print(page_info())\n" + "PY\n" + "```" + ) + result += "\n" + + return result + def get_servers_context(self): """ Generate a context block for available MCP servers. diff --git a/cecli/coders/base_coder.py b/cecli/coders/base_coder.py index a6714110d1c..deede113a49 100755 --- a/cecli/coders/base_coder.py +++ b/cecli/coders/base_coder.py @@ -4165,6 +4165,7 @@ async def show_send_output(self, completion): async def show_send_output_stream(self, completion): received_content = False chunk_index = 0 + track_loops = self.edit_format in ("agent", "subagent") content_detector = LoopDetector(self.repetition_penalties) tool_detector = LoopDetector(self.repetition_penalties) loop_detected = False @@ -4213,7 +4214,8 @@ async def show_send_output_stream(self, completion): tool_call_chunk.function.arguments ) - tool_detector.push(tool_call_chunk.function.arguments) + if track_loops: + tool_detector.push(tool_call_chunk.function.arguments) except (AttributeError, IndexError): # Handle cases where the response structure doesn't match expectations @@ -4279,7 +4281,7 @@ async def show_send_output_stream(self, completion): chunk._hidden_params["created_at"] = chunk_index self.partial_response_chunks.append(chunk) - if text: + if text and track_loops: content_detector.push(text) if self.show_pretty(): diff --git a/cecli/commands/web.py b/cecli/commands/web.py index 3de9f7b6cd7..b0c071a351e 100644 --- a/cecli/commands/web.py +++ b/cecli/commands/web.py @@ -2,8 +2,9 @@ from cecli.commands.utils.base_command import BaseCommand from cecli.commands.utils.helpers import format_command_result +from cecli.helpers.browser import is_browser_harness_available from cecli.helpers.conversation import ConversationService, MessageTag -from cecli.scrape import Scraper, install_playwright +from cecli.scrape import Scraper class WebCommand(BaseCommand): @@ -25,27 +26,12 @@ async def execute(cls, io, coder, args, **kwargs): scraper = kwargs.get("scraper") if not scraper: - # Get disable_playwright from coder args - disable_playwright = ( - getattr(coder.args, "disable_playwright", False) if coder and coder.args else False - ) - if disable_playwright: - res = False - else: - try: - res = await install_playwright(io) - if not res: - io.tool_warning("Unable to initialize playwright.") - except Exception: - io.tool_warning("Unable to initialize playwright.") - res = False - - # Get verify_ssl from kwargs or use default verify_ssl = kwargs.get("verify_ssl", True) scraper = Scraper( print_error=io.tool_error, - playwright_available=res, + browser_harness_available=is_browser_harness_available(), + browser_config=getattr(getattr(coder, "args", None), "browser", None), verify_ssl=verify_ssl, ) @@ -88,6 +74,7 @@ def get_help(cls) -> str: help_text += ( "\nThis command scrapes a webpage, converts it to markdown, and adds it to the chat.\n" ) - help_text += "It uses Playwright for JavaScript-rendered pages when available.\n" - help_text += "Use --disable-playwright to disable Playwright and use simpler scraping.\n" + help_text += ( + "It uses browser-harness for JavaScript-rendered pages and falls back to plain HTTP.\n" + ) return help_text diff --git a/cecli/helpers/agents/service.py b/cecli/helpers/agents/service.py index f1777b56971..6199efea240 100644 --- a/cecli/helpers/agents/service.py +++ b/cecli/helpers/agents/service.py @@ -406,6 +406,21 @@ def _get_tui(coder: Any) -> Any: # If it is already a plain reference (e.g., in tests), use it directly. return tui_ref + def _ensure_registry_loaded(self) -> None: + """Load sub-agent definitions when nothing has built the registry yet. + + Only ``AgentCoder`` bootstraps the registry at construction, so + non-agent coders (e.g. ask mode) would otherwise report every + sub-agent as unknown to commands such as ``/creator``. + """ + if self._global_registry: + return + + self.build_registry( + list(getattr(self.coder, "sub_agent_paths", None) or []), + root=getattr(self.coder, "root", None), + ) + def _resolve_model_sentinel(self, value: str, parent_coder: Any): """Resolve a sentinel model marker to the appropriate Model instance. @@ -713,6 +728,10 @@ async def _create_sub_agent_coder( ) config = self._global_registry.get(name) + if not config: + self._ensure_registry_loaded() + config = self._global_registry.get(name) + if not config: raise ValueError( f"Unknown sub-agent '{name}'. " f"Available: {list(self._global_registry.keys())}" diff --git a/cecli/helpers/browser.py b/cecli/helpers/browser.py new file mode 100644 index 00000000000..b97774c3439 --- /dev/null +++ b/cecli/helpers/browser.py @@ -0,0 +1,439 @@ +"""Browser-harness CDP integration helpers. + +browser-harness drives a running Chrome/Chromium over the Chrome DevTools +Protocol. It reads the endpoint from ``BU_CDP_URL``, so cecli detects or +launches a CDP-enabled browser and exports that variable before running a +browser-harness command. Browser settings can override the endpoint, launch +command, port, profile directory, and headless mode. +""" + +import http.client +import os +import platform +import re +import shutil +import subprocess +import tempfile +import time +import urllib.parse +from pathlib import Path + +BROWSER_HARNESS_COMMAND = "browser-harness" +CDP_URL_ENV = "BU_CDP_URL" +DEFAULT_CDP_PORT = 9222 +DEFAULT_CDP_PORTS = (9222, 9223, 9224, 9225) +DEFAULT_LAUNCH_TIMEOUT = 10.0 +LAUNCH_COOLDOWN_SECONDS = 30.0 + +_BROWSER_HARNESS_PATTERN = re.compile( + r"(?:^|[;&|()]\s*)" + r"(?:(?:sudo|doas|exec|env|nohup|time|nice|uvx)\s+|timeout\s+\d+\s+" + r"|[A-Za-z_][A-Za-z0-9_]*=\S*\s+)*" + r"browser[-_]harness\b", + re.MULTILINE, +) + +_last_launch_attempt = 0.0 + + +def is_browser_harness_available(): + """Return True when the browser-harness CLI is installed.""" + return shutil.which(BROWSER_HARNESS_COMMAND) is not None + + +def is_browser_harness_command(command): + """Return True when a shell command invokes the browser-harness CLI.""" + if not command: + return False + + return _BROWSER_HARNESS_PATTERN.search(command) is not None + + +def parse_browser_config(value): + """Parse a browser config mapping from a dict, JSON/YAML string, or file path.""" + import json + + import yaml + + if not value: + return {} + + if isinstance(value, dict): + return dict(value) + + if not isinstance(value, str): + return {} + + stripped = value.strip() + if not stripped: + return {} + + for parser in (json.loads, yaml.safe_load): + try: + parsed = parser(stripped) + except (json.JSONDecodeError, yaml.YAMLError, TypeError, ValueError): + continue + + if isinstance(parsed, dict): + return parsed + + try: + path = Path(stripped).expanduser() + except (OSError, RuntimeError, ValueError): + return {} + if path.is_file(): + try: + parsed = yaml.safe_load(path.read_text(encoding="utf-8")) + if isinstance(parsed, dict): + return parsed + except (OSError, yaml.YAMLError): + pass + + return {} + + +def get_browser_config(browser_config=None): + """Return normalized browser settings from config and the environment.""" + config = parse_browser_config(browser_config) + + return { + "cdp_url": config.get("cdp_url"), + "env_cdp_url": os.environ.get(CDP_URL_ENV), + "port": _as_int(config.get("port"), DEFAULT_CDP_PORT), + "user_data_dir": config.get("user_data_dir") or default_user_data_dir(), + "launch_command": config.get("launch_command"), + "launch": _as_bool(config.get("launch"), True), + "headless": _as_bool(config.get("headless"), False), + "launch_timeout": _as_float(config.get("launch_timeout"), DEFAULT_LAUNCH_TIMEOUT), + "harness_timeout": _as_float(config.get("harness_timeout"), 30.0), + } + + +def default_user_data_dir(): + """Return the persistent Chrome profile directory used for cecli launches.""" + configured = os.environ.get("CECLI_BROWSER_PROFILE_DIR") + if configured: + return configured + + try: + home = Path.home() + except RuntimeError: + # ``Path.home()`` raises when HOME/USERPROFILE are unset (for example in + # a stripped-down environment). Fall back to a temporary location so the + # browser helpers stay usable instead of aborting. + home = Path(tempfile.gettempdir()) + + return str(home / ".cecli" / "browser-profile") + + +def cdp_available(cdp_url, timeout=1.0): + """Return True when a Chrome DevTools endpoint answers at cdp_url.""" + if not cdp_url: + return False + + parsed = urllib.parse.urlparse(cdp_url) + if not parsed.hostname: + return False + + connection = http.client.HTTPConnection(parsed.hostname, parsed.port or 80, timeout=timeout) + path = parsed.path.rstrip("/") + "/json/version" + try: + connection.request("GET", path) + response = connection.getresponse() + return 200 <= response.status < 300 + except (OSError, http.client.HTTPException): + return False + finally: + connection.close() + + +def ensure_cdp_url(browser_config=None, launch=None, headless=None): + """Resolve a CDP endpoint, optionally launching a browser when none exists.""" + config = get_browser_config(browser_config) + + if config["cdp_url"]: + return config["cdp_url"] + + env_cdp_url = config["env_cdp_url"] + if env_cdp_url and cdp_available(env_cdp_url): + return env_cdp_url + + existing = find_existing_cdp_url(config) + if existing: + return existing + + if launch is None: + launch = config["launch"] + + launched = launch_cdp_browser(config, headless=headless) if launch else None + if launched: + return launched + + return env_cdp_url + + +def find_active_cdp_url(browser_config=None, include_default_ports=False): + """Return a live CDP endpoint without launching a browser.""" + config = get_browser_config(browser_config) + + if config["cdp_url"]: + return config["cdp_url"] + + env_cdp_url = config["env_cdp_url"] + if env_cdp_url and cdp_available(env_cdp_url, timeout=0.25): + return env_cdp_url + + ports = [config["port"]] + active_port = _read_active_port(config["user_data_dir"]) + if active_port: + ports.append(active_port) + if include_default_ports: + ports.extend(DEFAULT_CDP_PORTS) + + for port in dict.fromkeys(ports): + cdp_url = build_cdp_url(port=port) + if cdp_available(cdp_url, timeout=0.25): + return cdp_url + + return None + + +def export_active_cdp_url(browser_config=None, include_default_ports=False): + """Export BU_CDP_URL when a live browser already exists.""" + cdp_url = find_active_cdp_url(browser_config, include_default_ports=include_default_ports) + if cdp_url: + os.environ[CDP_URL_ENV] = cdp_url + + return cdp_url + + +def find_existing_cdp_url(config=None): + """Probe the configured port and common Chrome debugging ports.""" + config = config or get_browser_config() + ports = [config["port"]] + + active_port = _read_active_port(config["user_data_dir"]) + if active_port: + ports.append(active_port) + + ports.extend(DEFAULT_CDP_PORTS) + + seen = set() + for port in ports: + if port in seen: + continue + + seen.add(port) + cdp_url = build_cdp_url(port=port) + if cdp_available(cdp_url, timeout=0.4): + return cdp_url + + return None + + +def launch_cdp_browser(config=None, headless=None): + """Launch a CDP-enabled browser and wait for its DevTools endpoint.""" + global _last_launch_attempt + + config = config or get_browser_config() + now = time.monotonic() + if now - _last_launch_attempt < LAUNCH_COOLDOWN_SECONDS: + return None + + command, use_shell = build_launch_command(config, headless=headless) + if not command: + return None + + _last_launch_attempt = now + + try: + Path(config["user_data_dir"]).mkdir(parents=True, exist_ok=True) + subprocess.Popen( + command, + shell=use_shell, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + except OSError: + return None + + cdp_url = build_cdp_url(port=config["port"]) + deadline = time.monotonic() + config["launch_timeout"] + while time.monotonic() < deadline: + if cdp_available(cdp_url, timeout=0.5): + return cdp_url + + time.sleep(0.2) + + return None + + +def build_launch_command(config=None, headless=None): + """Return ``(command, use_shell)`` for launching a CDP-enabled browser.""" + config = config or get_browser_config() + use_headless = config["headless"] if headless is None else headless + + if config["launch_command"]: + try: + command = config["launch_command"].format( + port=config["port"], + user_data_dir=config["user_data_dir"], + profile_dir=config["user_data_dir"], + headless="--headless=new" if use_headless else "", + url="about:blank", + ) + except (KeyError, IndexError, ValueError): + command = config["launch_command"] + + return command, True + + executable = find_browser_executable() + if not executable: + return None, False + + command = [ + executable, + f"--remote-debugging-port={config['port']}", + f"--user-data-dir={config['user_data_dir']}", + "--no-first-run", + "--no-default-browser-check", + ] + if use_headless: + command.append("--headless=new") + + command.append("about:blank") + + return command, False + + +def find_browser_executable(): + """Return a known Chrome/Chromium/Edge executable path when available.""" + candidates = [ + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + "chrome", + "microsoft-edge", + "microsoft-edge-stable", + "brave-browser", + "vivaldi", + ] + + for candidate in candidates: + path = shutil.which(candidate) + if path: + return path + + absolute_candidates = _absolute_browser_candidates() + for candidate in absolute_candidates: + if os.path.isfile(candidate): + return candidate + + return None + + +def browser_harness_env(browser_config=None, headless=None): + """Return an environment with BU_CDP_URL set when a CDP browser is available.""" + env = os.environ.copy() + cdp_url = ensure_cdp_url(browser_config, headless=headless) + if cdp_url: + env[CDP_URL_ENV] = cdp_url + + return env + + +def configure_browser_environment(browser_config=None, command=None, headless=None): + """Set BU_CDP_URL in os.environ for browser-harness shell commands. + + When ``command`` is provided, browser-harness invocations trigger detection + or launch. Other commands still export an endpoint when a live browser + already exists, without opening a new one. Passing ``None`` always resolves + an endpoint, which is useful for the scraper. + """ + if command is not None and not is_browser_harness_command(command): + return export_active_cdp_url(browser_config) or os.environ.get(CDP_URL_ENV) + + cdp_url = ensure_cdp_url(browser_config, headless=headless) + if cdp_url: + os.environ[CDP_URL_ENV] = cdp_url + + return cdp_url + + +def build_cdp_url(host="127.0.0.1", port=DEFAULT_CDP_PORT): + """Build the HTTP DevTools endpoint URL for a host and port.""" + return f"http://{host}:{port}" + + +def _read_active_port(user_data_dir): + """Read the port Chrome writes to DevToolsActivePort in a profile dir.""" + try: + content = (Path(user_data_dir) / "DevToolsActivePort").read_text(encoding="utf-8") + return int(content.splitlines()[0].strip()) + except (OSError, ValueError, IndexError): + return None + + +def _absolute_browser_candidates(): + """Return platform-specific browser paths for common install locations.""" + system = platform.system() + + if system == "Darwin": + return [ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser", + ] + + if system == "Windows": + roots = [ + os.environ.get("PROGRAMFILES"), + os.environ.get("PROGRAMFILES(X86)"), + os.environ.get("LOCALAPPDATA"), + ] + relative = [ + ("Google", "Chrome", "Application", "chrome.exe"), + ("Chromium", "Application", "chrome.exe"), + ("Microsoft", "Edge", "Application", "msedge.exe"), + ("BraveSoftware", "Brave-Browser", "Application", "brave.exe"), + ] + + return [str(Path(root).joinpath(*parts)) for root in roots if root for parts in relative] + + return [ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/snap/bin/chromium", + "/usr/bin/microsoft-edge", + "/usr/bin/brave-browser", + ] + + +def _as_int(value, default): + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _as_float(value, default): + try: + return float(value) + except (TypeError, ValueError): + return default + + +def _as_bool(value, default): + if value is None: + return default + + if isinstance(value, str): + return value.strip().lower() not in ("0", "false", "no", "off") + + return bool(value) diff --git a/cecli/helpers/conversation/integration.py b/cecli/helpers/conversation/integration.py index fd42101cc02..4a795c0a435 100644 --- a/cecli/helpers/conversation/integration.py +++ b/cecli/helpers/conversation/integration.py @@ -797,7 +797,8 @@ def add_static_context_blocks(self) -> None: """ Add static context blocks to conversation (priority 50). - Static blocks include: environment_info, directory_structure, skills, servers, sub_agents + Static blocks include: environment_info, browser_automation, directory_structure, + skills, servers, sub_agents """ coder = self.get_coder() if not coder: @@ -817,6 +818,10 @@ def add_static_context_blocks(self) -> None: block = coder.get_cached_context_block("environment_info") if block: message_blocks["environment_info"] = block + if "browser_automation" in coder.allowed_context_blocks: + block = coder.get_cached_context_block("browser_automation") + if block: + message_blocks["browser_automation"] = block if "directory_structure" in coder.allowed_context_blocks: block = coder.get_cached_context_block("directory_structure") if block: diff --git a/cecli/helpers/key_detection.py b/cecli/helpers/key_detection.py index 8c3cb73fbc0..76e26ed7fc5 100644 --- a/cecli/helpers/key_detection.py +++ b/cecli/helpers/key_detection.py @@ -1,8 +1,6 @@ -import math import os import re import threading -from collections import Counter from typing import Any, Iterable, List, Mapping, Optional, Set # A run of key-safe characters (letters, digits and -_. only, so never any @@ -39,11 +37,13 @@ "apikeys", "credential", "credentials", + "jwt", "key", "keys", "passwd", "password", "passwords", + "private", "secret", "secrets", "token", @@ -61,27 +61,45 @@ separator + suffix for separator in _SUFFIX_SEPARATORS for suffix in SECRET_NAME_SUFFIXES ) -# A name that contains a credential word that is not a suffix ("KEY_DESCRIPTION_V2") -# is not trusted on its own: the value must also look key-like, measured as -# Shannon entropy in bits per character. Passwords are allowed a lower bar -# because they are often human-chosen rather than random. -DEFAULT_CREDENTIAL_ENTROPY = 3.2 - -CREDENTIAL_ENTROPY_THRESHOLDS = { - "apikey": 3.2, - "apikeys": 3.2, - "credential": 3.0, - "credentials": 3.0, - "key": 3.2, - "keys": 3.2, - "passwd": 2.5, - "password": 2.5, - "passwords": 2.5, - "secret": 3.0, - "secrets": 3.0, - "token": 3.2, - "tokens": 3.2, -} +# Prefixes used by well-known providers. A value starting with one of these is +# key-shaped without needing any other check. +KNOWN_KEY_PREFIXES = ( + "sk-ant-", + "sk-proj-", + "sk-", + "AIzaSy", + "AKIA", + "ASIA", + "ghp_", + "gho_", + "ghs_", + "ghr_", + "github_pat_", + "glpat-", + "xoxa-", + "xoxb-", + "xoxp-", + "xoxr-", + "xoxs-", + "gsk_", + "hf_", + "r8_", + "sk_live_", + "rk_live_", + "SG.", + "eyJ", +) + +# A short run of same-case letters followed by a separator (``ghp_``, ``sk-``, +# ``LLM_``) marks a key-shaped value even when the rest is unknown. +_KEY_SHAPE_PREFIX_RE = re.compile(r"^(?:[A-Z]{2,5}|[a-z]{2,5})[-_]") + +# Two or more letter/digit transitions, e.g. ``a2bc3`` or ``1de456f``. Prose +# and identifiers rarely alternate runs this way. +_ALTERNATING_RUN_RE = re.compile(r"[A-Za-z]+\d+[A-Za-z]+|\d+[A-Za-z]+\d+") + +# Values starting with these are paths, identifiers rather than secrets. +_PATH_START_PREFIXES = ("/", "\\", "./", ".\\", "../", "..\\", "~/", "~\\", "~") # Name words that mark a value as an identifier (paths, models, session ids, # hostnames). These are never scanned for key-like tokens: config/env entries @@ -163,8 +181,8 @@ def detect_api_keys( punctuation that a key-like token regex cannot span. - containing a credential word, as a word or inside a longer token (e.g. "KEY_DESCRIPTION_V2" or "sortKey"): the whole value is taken only when - it also looks key-like (length, letter/digit mix and entropy above the - threshold for that word). + it also looks key-like (length, mixed case, alternating letter/digit + runs or a known provider prefix). - identifier-named (see BENIGN_NAME_WORDS) or path-shaped: not registered in full, because these legitimately hold long non-secret strings; any key-like tokens embedded in the value are still registered. @@ -301,7 +319,7 @@ def _scan_entry(name: Any, value: Any, found: Set[str], expand: bool, forced: bo return if _contains_secret_word(name) or _is_ambiguous_secret_name(name): - if _looks_like_key_value(text, name): + if _looks_like_key_value(text): _register_secret_value(text, found) return @@ -380,38 +398,51 @@ def _contains_secret_word(name: Any) -> bool: return bool(_name_words(name) & SECRET_NAME_WORDS) -def _looks_like_key_value(value: str, name: Any) -> bool: +def _looks_like_key_value(value: str) -> bool: """True if a value under a containing credential name looks like a key. - Best-effort: requires ``MIN_KEY_LENGTH`` characters, letters and digits and - no whitespace, so short or all-alpha passphrases under ambiguous names are - intentionally not detected. + Path-starting and whitespace-containing values are rejected. Otherwise the + value must match at least one key tell: a known provider prefix, a short + same-case prefix followed by ``-``/``_``, mixed upper/lowercase without path + separators, or an alternating letter/digit run. """ stripped = value.strip() - if len(stripped) < MIN_KEY_LENGTH or _looks_like_path(stripped): + if len(stripped) < MIN_KEY_LENGTH or _starts_with_path(stripped): return False if any(char.isspace() for char in stripped): return False - has_alpha = any(char.isalpha() for char in stripped) - has_digit = any(char.isdigit() for char in stripped) + if stripped.startswith(KNOWN_KEY_PREFIXES): + return True + + if _KEY_SHAPE_PREFIX_RE.match(stripped): + return True - if not has_alpha or not has_digit: + if "/" in stripped or "\\" in stripped: return False - return _shannon_entropy(stripped) >= _entropy_threshold(name) + if _has_mixed_case(stripped): + return True + + return bool(_ALTERNATING_RUN_RE.search(stripped)) + + +def _has_mixed_case(value: str) -> bool: + """True if ``value`` contains both uppercase and lowercase letters.""" + has_upper = any(char.isupper() for char in value) + has_lower = any(char.islower() for char in value) + return has_upper and has_lower -def _entropy_threshold(name: Any) -> float: - """Return the entropy bar for the credential words in ``name``.""" - words = _name_words(name) & set(CREDENTIAL_ENTROPY_THRESHOLDS) - if not words: - return DEFAULT_CREDENTIAL_ENTROPY +def _starts_with_path(value: str) -> bool: + """True if ``value`` begins like an absolute, relative or home path.""" + if _WINDOWS_PATH_RE.match(value): + return True - return min(CREDENTIAL_ENTROPY_THRESHOLDS[word] for word in words) + return value.startswith(_PATH_START_PREFIXES) def _looks_like_path(value: str) -> bool: @@ -476,14 +507,6 @@ def _register_embedded_keys(value: str, found: Set[str]) -> None: found.update(_KEY_LIKE_RE.findall(chunk)) -def _shannon_entropy(value: str) -> float: - """Return the Shannon entropy of ``value`` in bits per character.""" - counts = Counter(value) - length = len(value) - - return -sum((count / length) * math.log2(count / length) for count in counts.values()) - - def _redact_mapping(mapping: Mapping) -> Mapping: """Apply redaction to every value, keeping the mapping when unchanged.""" items = {} diff --git a/cecli/main.py b/cecli/main.py index bd210673b04..51b7047778b 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -162,6 +162,7 @@ def convert_yaml_to_json_string(value, config_file_value=None): # the same option, so CLI keys win per-key while file-only keys are preserved. YAML_TO_JSON_ARG_KEYS = { "agent_config": "agent-config", + "browser": "browser", "tui_config": "tui-config", "mcp_servers": "mcp-servers", "custom": "custom", @@ -1403,7 +1404,7 @@ def get_io(pretty): main_model=main_model, # Agent mode is the startup default; an explicit flag/config/env value # still wins, and reloads keep the session's current mode. - edit_format=args.edit_format or "agent", + edit_format=args.edit_format or (from_coder.edit_format if from_coder else "agent"), io=io, args=args, repo=repo, diff --git a/cecli/scrape.py b/cecli/scrape.py index f1a6b03ae83..66307358d84 100755 --- a/cecli/scrape.py +++ b/cecli/scrape.py @@ -6,93 +6,25 @@ import pypandoc -from cecli import __version__, urls, utils +from cecli import __version__, urls from cecli.dump import dump # noqa: F401 +from cecli.helpers.browser import is_browser_harness_available coder_user_agent = f"cecli/{__version__} +{urls.website}" -# Playwright is nice because it has a simple way to install dependencies on most -# platforms. - - -def check_playwright(): - try: - from playwright.async_api import async_playwright # noqa: F401 - from playwright.sync_api import sync_playwright # noqa: F401 - - has_playwright = True - except ImportError: - has_playwright = False - - return has_playwright - - -async def check_chromium(): - has_chromium = False - - if check_playwright(): - from playwright.async_api import async_playwright - - try: - async with async_playwright() as p: - browser = await p.chromium.launch() - await browser.close() - has_chromium = True - except Exception as e: - has_chromium = False - print(f"chromium errors {e}") - - return has_chromium - - -async def install_playwright(io): - has_playwright = check_playwright() - has_chromium = await check_chromium() - - if has_playwright and has_chromium: - return True - - pip_cmd = utils.get_pip_install(["cecli-dev[playwright]"]) - chromium_cmd = "-m playwright install --with-deps chromium" - chromium_cmd = [sys.executable] + chromium_cmd.split() - - cmds = "" - if not has_playwright: - cmds += " ".join(pip_cmd) + "\n" - if not has_chromium: - cmds += " ".join(chromium_cmd) + "\n" - - text = f"""For the best web scraping, install Playwright: - -{cmds} -See {urls.enable_playwright} for more info. -""" - - io.tool_output(text) - if not await io.confirm_ask("Install playwright?", default="y"): - return - - if not has_playwright: - success, output = utils.run_install(pip_cmd) - if not success: - io.tool_error(output) - return - - success, output = utils.run_install(chromium_cmd) - if not success: - io.tool_error(output) - return - - return True - class Scraper: pandoc_available = None - playwright_available = None - playwright_instructions_shown = False + _browser_harness_tab_ready = False # Public API... - def __init__(self, print_error=None, playwright_available=None, verify_ssl=True): + def __init__( + self, + print_error=None, + verify_ssl=True, + browser_harness_available=None, + browser_config=None, + ): """ `print_error` - a function to call to print error/debug info. `verify_ssl` - if False, disable SSL certificate verification when scraping. @@ -102,8 +34,12 @@ def __init__(self, print_error=None, playwright_available=None, verify_ssl=True) else: self.print_error = print - self.playwright_available = playwright_available self.verify_ssl = verify_ssl + if browser_harness_available is None: + browser_harness_available = is_browser_harness_available() + + self.browser_harness_available = browser_harness_available + self.browser_config = browser_config or {} async def scrape(self, url): """ @@ -113,9 +49,13 @@ async def scrape(self, url): `url` - the URL to scrape. """ - if self.playwright_available: - content, mime_type = await self.scrape_with_playwright(url) - else: + content = None + mime_type = None + + if self.browser_harness_available: + content, mime_type = await self.scrape_with_browser_harness(url) + + if not content: content, mime_type = self.scrape_with_httpx(url) if not content: @@ -152,54 +92,105 @@ def looks_like_html(self, content): return False # Internals... - async def scrape_with_playwright(self, url): - import playwright # noqa: F401 - from playwright.async_api import Error as PlaywrightError - from playwright.async_api import TimeoutError as PlaywrightTimeoutError - from playwright.async_api import async_playwright + async def scrape_with_browser_harness(self, url): + """Render a page through browser-harness and return its outer HTML.""" + import json + import os + import tempfile + + from cecli.helpers.browser import ( + browser_harness_env, + get_browser_config, + is_browser_harness_available, + ) + + if not is_browser_harness_available(): + return None, None + + config = get_browser_config(self.browser_config) + output_fd, output_path = tempfile.mkstemp(prefix="cecli-browser-", suffix=".json") + os.close(output_fd) + + url_literal = json.dumps(url) + output_literal = json.dumps(output_path) + reuse_tab = Scraper._browser_harness_tab_ready + + script = f""" +import json +import pathlib + +url = {url_literal} +reuse_tab = {reuse_tab!r} + +try: + if reuse_tab: + ensure_real_tab() + goto_url(url) + else: + new_tab(url) +except Exception: + new_tab(url) + +wait_for_load() + +data = {{ + "contentType": js("document.contentType"), + "html": js("document.documentElement.outerHTML"), +}} +pathlib.Path({output_literal}).write_text(json.dumps(data), encoding="utf-8") +""" + + process = None + try: + env = await asyncio.to_thread(browser_harness_env, self.browser_config, True) + process = await asyncio.create_subprocess_exec( + "browser-harness", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + _stdout, stderr = await asyncio.wait_for( + process.communicate(script.encode("utf-8")), + timeout=config["harness_timeout"], + ) + + if process.returncode != 0: + error_text = stderr.decode("utf-8", errors="replace").strip() + if error_text: + self.print_error(f"browser-harness scrape failed: {error_text[-1000:]}") - async with async_playwright() as p: - try: - browser = await p.chromium.launch() - except Exception as e: - self.playwright_available = False - self.print_error(str(e)) return None, None + with open(output_path, encoding="utf-8") as output_file: + data = json.load(output_file) + except asyncio.TimeoutError: + if process is not None: + process.kill() + await process.wait() + + self.print_error(f"browser-harness timed out scraping {url}") + return None, None + except (OSError, ValueError): + return None, None + finally: try: - context = await browser.new_context(ignore_https_errors=not self.verify_ssl) - page = await context.new_page() - - user_agent = await page.evaluate("navigator.userAgent") - user_agent = user_agent.replace("Headless", "") - user_agent = user_agent.replace("headless", "") - user_agent += " " + coder_user_agent - - await page.set_extra_http_headers({"User-Agent": user_agent}) - - response = None - try: - response = await page.goto(url, wait_until="networkidle", timeout=5000) - except PlaywrightTimeoutError: - self.print_error(f"Page didn't quiesce, scraping content anyway: {url}") - response = None - except PlaywrightError as e: - self.print_error(f"Error navigating to {url}: {str(e)}") - return None, None - - try: - content = await page.content() - mime_type = None - if response: - content_type = await response.header_value("content-type") - if content_type: - mime_type = content_type.split(";")[0] - except PlaywrightError as e: - self.print_error(f"Error retrieving page content: {str(e)}") - content = None - mime_type = None - finally: - await browser.close() + os.unlink(output_path) + except OSError: + pass + + content = data.get("html") or "" + mime_type = data.get("contentType") or None + + if not content: + return None, None + + if mime_type and not ( + mime_type.startswith("text/html") or mime_type.startswith("application/xhtml") + ): + return None, None + + Scraper._browser_harness_tab_ready = True return content, mime_type @@ -282,7 +273,11 @@ def slimdown_html(soup): async def main(url): - scraper = Scraper(playwright_available=check_playwright()) + from cecli.helpers.browser import is_browser_harness_available + + scraper = Scraper( + browser_harness_available=is_browser_harness_available(), + ) content = await scraper.scrape(url) print(content) diff --git a/cecli/tools/shell.py b/cecli/tools/shell.py index c0c5b12feff..04b183b1381 100644 --- a/cecli/tools/shell.py +++ b/cecli/tools/shell.py @@ -16,6 +16,7 @@ import xxhash from cecli.helpers.background_commands import BackgroundCommandManager +from cecli.helpers.browser import configure_browser_environment from cecli.run_cmd import run_cmd, run_cmd_subprocess from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError, is_creator_coder @@ -237,7 +238,10 @@ async def execute( response.append_result("Command execution skipped by user.") return response + browser_command = command command = coder.format_command_with_prefix(command) + browser_config = getattr(getattr(coder, "args", None), "browser", None) + configure_browser_environment(browser_config, command=browser_command) # Determine timeout from agent_config (default: 30 seconds) as fallback config_timeout = 0 diff --git a/cecli/urls.py b/cecli/urls.py index f5cb615774e..f90eb74080d 100644 --- a/cecli/urls.py +++ b/cecli/urls.py @@ -1,7 +1,6 @@ website = "https://cecli.dev/" edit_errors = "https://cecli.dev/docs/troubleshooting/edit-errors/" git = "https://cecli.dev/docs/git/" -enable_playwright = "https://cecli.dev/docs/usage/optional/#enable-playwright" favicon = "https://cecli.dev/assets/cecli-temp-logo-favicon.svg" model_warnings = "https://cecli.dev/docs/llms/warnings/" token_limits = "https://cecli.dev/docs/troubleshooting/token-limits/" diff --git a/cecli/website/assets/sample.cecli.conf.yml b/cecli/website/assets/sample.cecli.conf.yml index a0275eee566..d10627af942 100644 --- a/cecli/website/assets/sample.cecli.conf.yml +++ b/cecli/website/assets/sample.cecli.conf.yml @@ -92,7 +92,7 @@ ## Timeout in seconds for API calls (default: None) #timeout: xxx -## Specify what edit format the LLM should use (default depends on model) +## Specify what edit format the LLM should use (default: agent) #edit-format: xxx ## Use architect edit format for the main chat @@ -368,10 +368,14 @@ #voice-input-device: xxx ################# -# Other settings: +################ +# Browser Settings: + +## Specify browser-harness configuration as a JSON/YAML string or file path +#browser: xxx -## Never prompt for or attempt to install Playwright for web scraping (default: False). -#disable-playwright: false +################# +# Other settings: ## specify a file to edit (can be used multiple times) #file: xxx diff --git a/cecli/website/assets/sample.env b/cecli/website/assets/sample.env index 1749fe0ac05..0b3dadc698c 100644 --- a/cecli/website/assets/sample.env +++ b/cecli/website/assets/sample.env @@ -81,7 +81,7 @@ ## Timeout in seconds for API calls (default: None) #AIDER_TIMEOUT= -## Specify what edit format the LLM should use (default depends on model) +## Specify what edit format the LLM should use (default: agent) #AIDER_EDIT_FORMAT= ## Use architect edit format for the main chat @@ -354,8 +354,8 @@ ################# # Other settings: -## Never prompt for or attempt to install Playwright for web scraping (default: False). -#AIDER_DISABLE_PLAYWRIGHT=false +## Specify browser-harness configuration as a JSON/YAML string or file path +#CECLI_BROWSER= ## specify a file to edit (can be used multiple times) #AIDER_FILE= diff --git a/cecli/website/docs/config/agent-mode.md b/cecli/website/docs/config/agent-mode.md index 11d3339baae..d6cede08849 100644 --- a/cecli/website/docs/config/agent-mode.md +++ b/cecli/website/docs/config/agent-mode.md @@ -300,6 +300,7 @@ The following context blocks are available and can be customized using `include_ **Included by default:** - **`environment_info`**: Working directory, platform, date, language, and repository details +- **`browser_automation`**: browser-harness usage guidance, included when browser-harness is installed - **`todo_list`**: Current tasks managed through `UpdateTodoList` - **`skills`**: Available skills and their configuration - **`servers`**: Connected, filtered, and disconnected MCP servers @@ -379,6 +380,7 @@ agent-config: This configuration system allows for fine-grained control over which tools are available in Agent Mode, enabling security-conscious deployments and specialized workflows while maintaining essential functionality. + ### Skills Agent Mode includes a powerful skills system that allows you to extend the AI's capabilities with custom instructions, reference materials, scripts, and assets. Skills are configured through the `agent-config` parameter in the YAML configuration file. diff --git a/cecli/website/docs/config/browser.md b/cecli/website/docs/config/browser.md new file mode 100644 index 00000000000..d6e2a7cca48 --- /dev/null +++ b/cecli/website/docs/config/browser.md @@ -0,0 +1,108 @@ +--- +parent: Configuration +nav_order: 16 +description: Connect cecli to a real browser for automation and JavaScript-rendered pages. +--- + +# Browser + +cecli can drive a real Chrome or Chromium browser through the +[browser-harness](https://github.com/browser-use/browser-harness) CLI. This is useful when a page +needs JavaScript, an existing login, or normal browser interaction. The `/web` command also uses it +to scrape JavaScript-rendered pages before falling back to plain HTTP. + +browser-harness ships with cecli on Python 3.11 and newer. You only need a supported browser +installed on the machine. On older Python versions, `/web` falls back to plain HTTP. + +## Quick start + +Most users do not need to configure anything. When the agent runs a browser-harness command, cecli +connects to a browser that is already available for automation. If none is found, it launches a +dedicated Chrome profile at `~/.cecli/browser-profile`. Whenever a live browser is already running, +cecli exports `BU_CDP_URL` so shell commands and browser-harness can attach to it. + +To connect to a browser you are already running, start Chrome with remote debugging enabled: + +```bash +google-chrome --remote-debugging-port=9222 --user-data-dir="$HOME/.cecli/browser-profile" +``` + +You can also open `chrome://inspect/#remote-debugging` and allow remote debugging for the current +browser instance. + +Then tell cecli to use it without launching another browser: + +```yaml +browser: + cdp_url: http://127.0.0.1:9222 + launch: false +``` + +## Configuration + +Set browser options with the `--browser` command-line option or the top-level `browser` key in a +config file. `--browser` accepts a JSON/YAML string or a path to a file: + +```bash +cecli --browser '{"cdp_url": "http://127.0.0.1:9222", "launch": false}' +``` + +## Defaults + +| Option | Default | Description | +|--------|---------|-------------| +| `cdp_url` | not set | Connect to this browser instead of searching for one. | +| `port` | `9222` | Port used when cecli launches a browser. | +| `user_data_dir` | `~/.cecli/browser-profile` | Profile directory used for launched browsers. | +| `launch_command` | auto-detected | Custom browser launch command. | +| `launch` | `true` | Launch a browser when a browser-harness command needs one. | +| `headless` | `false` | Run launched browsers without a visible window. The scraper always requests a headless launch. | +| `launch_timeout` | `10` | Seconds to wait for a launched browser to become ready. | +| `harness_timeout` | `30` | Seconds allowed for a browser-harness scrape. | + +cecli looks for common Chrome, Chromium, Edge, Brave, and Vivaldi installations. It also checks the +usual local debugging ports when `cdp_url` is not set. + +## Examples + +### Use a dedicated profile on another port + +```yaml +browser: + port: 9333 + user_data_dir: ~/.cecli/browser-profile-work +``` + +### Run launched browsers headless + +```yaml +browser: + headless: true +``` + +### Use a custom launch command + +```yaml +browser: + launch_command: >- + google-chrome --remote-debugging-port={port} + --user-data-dir={user_data_dir} --headless=new +``` + +The launch command may use `{port}`, `{user_data_dir}` or `{profile_dir}`, `{headless}`, and +`{url}` placeholders. + +### Never launch a browser automatically + +```yaml +browser: + launch: false +``` + +With `launch: false`, cecli only uses an existing browser. If none is available, browser-harness +commands fail instead of opening a new window. + +## Scraping + +`/web` uses browser-harness to render pages, then falls back to plain HTTP when browser-harness +cannot retrieve the page. \ No newline at end of file diff --git a/cecli/website/docs/config/conf.md b/cecli/website/docs/config/conf.md index 629473bfcc5..e2c91b31ffa 100644 --- a/cecli/website/docs/config/conf.md +++ b/cecli/website/docs/config/conf.md @@ -105,6 +105,7 @@ The following fields are stored as JSON/YAML strings but are **internally deep-m - `retries` - Retry configuration - `custom` - Custom configurations - `tui-config` - TUI configuration +- `browser` - Browser automation settings ### Examples @@ -280,7 +281,7 @@ Below is a sample of the YAML config file, which you can also [download from Git ## Timeout in seconds for API calls (default: None) #timeout: xxx -## Specify what edit format the LLM should use (default depends on model) +## Specify what edit format the LLM should use (default: agent) #edit-format: xxx ## Use architect edit format for the main chat @@ -567,12 +568,15 @@ Below is a sample of the YAML config file, which you can also [download from Git ## Specify the input device name for voice recording #voice-input-device: xxx +################ +# Browser Settings: + +## Specify browser-harness configuration as a JSON/YAML string or file path +#browser: xxx + ################# # Other settings: -## Never prompt for or attempt to install Playwright for web scraping (default: False). -#disable-playwright: false - ## specify a file to edit (can be used multiple times) #file: xxx ## Specify multiple values like this: diff --git a/cecli/website/docs/config/dotenv.md b/cecli/website/docs/config/dotenv.md index 05eba0961ea..cca9fd57a23 100644 --- a/cecli/website/docs/config/dotenv.md +++ b/cecli/website/docs/config/dotenv.md @@ -108,7 +108,7 @@ Below is a sample `.env` file, which you can also [download from GitHub](https:/ ## Timeout in seconds for API calls (default: None) #CECLI_TIMEOUT= -## Specify what edit format the LLM should use (default depends on model) +## Specify what edit format the LLM should use (default: agent) #CECLI_EDIT_FORMAT= ## Use architect edit format for the main chat @@ -378,8 +378,8 @@ Below is a sample `.env` file, which you can also [download from GitHub](https:/ ################# # Other settings: -## Never prompt for or attempt to install Playwright for web scraping (default: False). -#CECLI_DISABLE_PLAYWRIGHT=false +## Specify browser-harness configuration as a JSON/YAML string or file path +#CECLI_BROWSER= ## specify a file to edit (can be used multiple times) #CECLI_FILE= diff --git a/cecli/website/docs/config/options.md b/cecli/website/docs/config/options.md index 19268329662..d75b8cad457 100644 --- a/cecli/website/docs/config/options.md +++ b/cecli/website/docs/config/options.md @@ -66,11 +66,11 @@ usage: cecli [-h] [--model] [--openai-api-key] [--anthropic-api-key] [--show-release-notes | --no-show-release-notes] [--install-main-branch] [--upgrade] [--version] [--message] [--message-file] - [--gui | --no-gui | --browser | --no-browser] + [--browser BROWSER_CONFIG_JSON] [--copy-paste | --no-copy-paste] [--apply] [--apply-clipboard-edits] [--exit] [--show-repo-map] [--show-prompts] [--voice-format] [--voice-language] - [--voice-input-device] [--disable-playwright] [--file] + [--voice-input-device] [--file] [--read] [--vim] [--chat-language] [--commit-language] [--yes-always] [-v] [--load] [--encoding] [--line-endings] [-c] [--env-file] @@ -158,7 +158,7 @@ Verify the SSL cert when connecting to models (default: True) Default: True Envi Timeout in seconds for API calls (default: None) Environment variable: `CECLI_TIMEOUT` ### `--edit-format EDIT_FORMAT` -Specify what edit format the LLM should use (default depends on model) Environment variable: `CECLI_EDIT_FORMAT` Aliases: +Specify what edit format the LLM should use (default: agent) Environment variable: `CECLI_EDIT_FORMAT` Aliases: - `--edit-format EDIT_FORMAT` - `--chat-mode EDIT_FORMAT` @@ -482,8 +482,8 @@ Specify the input device name for voice recording Environment variable: `CECLI_V ## Other settings: -### `--disable-playwright` -Never prompt for or attempt to install Playwright for web scraping (default: False). Default: False Environment variable: `CECLI_DISABLE_PLAYWRIGHT` +### `--browser BROWSER_CONFIG_JSON` +Specify browser-harness configuration as a JSON/YAML string or file path Environment variable: `CECLI_BROWSER` ### `--file FILE` specify a file to edit (can be used multiple times) Environment variable: `CECLI_FILE` diff --git a/cecli/website/docs/usage/modes.md b/cecli/website/docs/usage/modes.md index 51ca0b3fa60..e63991b66ba 100644 --- a/cecli/website/docs/usage/modes.md +++ b/cecli/website/docs/usage/modes.md @@ -14,7 +14,7 @@ Cecli has a few different chat modes: - `help` - cecli will answer questions about cecli: usage, configuration, troubleshooting, etc. - `agent` - cecli will autonomously explore and modify your codebase using local tools, discovering and managing the relevant files itself. -By default, cecli starts in "code" mode. As you are talking, you can send individual messages in a specific mode using `/code`, `/architect`, `/ask`, `/help`, and `/agent` commands: Using these `/`-commands applies just to that particular message. Your next message will go back to the active mode (usually "code" mode by default). +By default, cecli starts in "agent" mode. As you are talking, you can send individual messages in a specific mode using `/code`, `/architect`, `/ask`, `/help`, and `/agent` commands: Using these `/`-commands applies just to that particular message. Your next message will go back to the active mode (usually "agent" mode by default). You can switch the active mode in a sticky way with the `/chat-mode ` command: diff --git a/cecli/website/docs/usage/optional.md b/cecli/website/docs/usage/optional.md index 5a02ad0adf7..8c09a1fcb91 100644 --- a/cecli/website/docs/usage/optional.md +++ b/cecli/website/docs/usage/optional.md @@ -23,19 +23,11 @@ You need an key from an API provider to work with most models: You can [store your api keys in configuration or env files](../config/api-keys.html) and they will be loaded automatically whenever you run cecli. -## Enable Playwright +## Enable Browser automation Cecli supports adding web pages to the chat with the `/web ` command. When you add a url to the chat, cecli fetches the page and scrapes its content. -By default, cecli uses the `httpx` library to scrape web pages, but this only works on a subset of web pages. Some sites explicitly block requests from tools like httpx. Others rely heavily on javascript to render the page content, which isn't possible using only httpx. - -Cecli works best with all web pages if you install Playwright's chromium browser and its dependencies: - -``` -playwright install --with-deps chromium -``` - -See the [Playwright for Python documentation](https://playwright.dev/python/docs/browsers#install-system-dependencies) for additional information. +Cecli uses browser-harness, which ships with cecli, to render pages that rely on JavaScript. You only need a Chrome or Chromium based browser installed. If no browser can be used, cecli falls back to plain HTTP requests. ## Enable voice coding diff --git a/docker/Dockerfile b/docker/Dockerfile index 71697e19c57..0ab2aaca745 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -23,14 +23,10 @@ RUN python -m venv /venv ENV PATH="/venv/bin:$PATH" RUN /venv/bin/python -m pip install --no-cache-dir uv -# Playwright browser settings -ENV PLAYWRIGHT_BROWSERS_PATH=/home/appuser/pw-browsers -ENV PLAYWRIGHT_SKIP_BROWSER_GC=1 - # Create directories with proper permissions -RUN mkdir -p /home/appuser/.cecli /home/appuser/.cache /home/appuser/pw-browsers && \ +RUN mkdir -p /home/appuser/.cecli /home/appuser/.cache && \ chown -R appuser:appuser /home/appuser /app /venv && \ - chmod -R 777 /home/appuser/.cecli /home/appuser/.cache /home/appuser/pw-browsers + chmod -R 777 /home/appuser/.cecli /home/appuser/.cache # So git doesn't complain about unusual permissions RUN git config --system --add safe.directory /app @@ -53,10 +49,6 @@ COPY requirements/ /tmp/cecli/requirements/ RUN uv pip install --no-cache-dir -r /tmp/cecli/requirements.txt && \ rm -rf /tmp/cecli -# Install playwright browsers -RUN uv pip install --no-cache-dir playwright && \ - /venv/bin/python -m playwright install --with-deps chromium - # Fix site-packages permissions RUN find /venv/lib/python3.12/site-packages \( -type d -exec chmod a+rwx {} + \) -o \( -type f -exec chmod a+rw {} + \) diff --git a/docker/Dockerfile.local.nvidia.cuda.ubuntu b/docker/Dockerfile.local.nvidia.cuda.ubuntu index 2c31c818aa0..12f3938819b 100644 --- a/docker/Dockerfile.local.nvidia.cuda.ubuntu +++ b/docker/Dockerfile.local.nvidia.cuda.ubuntu @@ -29,14 +29,11 @@ RUN python -m venv /venv ENV PATH="/venv/bin:$PATH" RUN /venv/bin/python -m pip install --no-cache-dir uv -# Playwright browser settings -ENV PLAYWRIGHT_BROWSERS_PATH=${OLDHOME}/pw-browsers -ENV PLAYWRIGHT_SKIP_BROWSER_GC=1 # Create directories with proper permissions -RUN mkdir -p ${OLDHOME}/.cecli ${OLDHOME}/.cache ${OLDHOME}/pw-browsers && \ +RUN mkdir -p ${OLDHOME}/.cecli ${OLDHOME}/.cache && \ chown -R ${DEFAULTUSER}:${DEFAULTGROUP} ${OLDHOME} /app /venv && \ - chmod -R 777 ${OLDHOME}/.cecli ${OLDHOME}/.cache ${OLDHOME}/pw-browsers + chmod -R 777 ${OLDHOME}/.cecli ${OLDHOME}/.cache # So git doesn't complain about unusual permissions RUN git config --system --add safe.directory /app @@ -54,9 +51,6 @@ COPY requirements/ /tmp/cecli/requirements/ RUN uv pip install --no-cache-dir -r /tmp/cecli/requirements.txt && \ rm -rf /tmp/cecli -# Install playwright browsers -RUN uv pip install --no-cache-dir playwright && \ - /venv/bin/python -m playwright install --with-deps chromium # Fix site-packages permissions RUN find /venv/lib/python3.12/site-packages \( -type d -exec chmod a+rwx {} + \) -o \( -type f -exec chmod a+rw {} + \) diff --git a/pyproject.toml b/pyproject.toml index 74808f90a7a..2f9e2235e3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ dependencies = { file = "requirements/requirements.in" } [tool.setuptools.dynamic.optional-dependencies] dev = { file = "requirements/requirements-dev.in" } help = { file = "requirements/requirements-help.in" } -playwright = { file = "requirements/requirements-playwright.in" } [tool.setuptools] include-package-data = true @@ -43,7 +42,7 @@ include = ["cecli*"] [tool.uv] exclude-newer = "2 days" -exclude-newer-package = { "py-cymbal" = "0 days" } +exclude-newer-package = { "py-cymbal" = "0 days", "cecli-browser-harness" = "0 days" } [build-system] requires = ["setuptools>=68", "setuptools_scm[toml]>=8"] diff --git a/requirements.txt b/requirements.txt index 9df58356241..e434fc88a6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,6 +46,14 @@ build==1.3.0 # via # -c requirements/common-constraints.txt # chromadb +cdp-use==1.4.5 + # via + # -c requirements/common-constraints.txt + # cecli-browser-harness +cecli-browser-harness==0.1.13.post1 + # via + # -c requirements/common-constraints.txt + # -r requirements/requirements.in certifi==2025.11.12 # via # -c requirements/common-constraints.txt @@ -98,6 +106,10 @@ durationpy==0.10 # via # -c requirements/common-constraints.txt # kubernetes +fetch-use==0.4.0 + # via + # -c requirements/common-constraints.txt + # cecli-browser-harness filelock==3.20.0 # via # -c requirements/common-constraints.txt @@ -157,6 +169,7 @@ httptools==0.8.0 httpx==0.28.1 # via # -c requirements/common-constraints.txt + # cdp-use # chromadb # mcp httpx-sse==0.4.3 @@ -323,10 +336,11 @@ pexpect==4.9.0 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in -pillow==12.0.0 +pillow==12.3.0 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in + # cecli-browser-harness platformdirs==4.5.0 # via # -c requirements/common-constraints.txt @@ -580,6 +594,7 @@ typing-extensions==4.15.0 # aiosignal # anyio # beautifulsoup4 + # cdp-use # chromadb # grpcio # huggingface-hub @@ -632,10 +647,12 @@ websocket-client==1.9.0 # via # -c requirements/common-constraints.txt # kubernetes -websockets==16.1.1 +websockets==17.4 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in + # cdp-use + # cecli-browser-harness # uvicorn xxhash==3.6.0 # via diff --git a/requirements/common-constraints.txt b/requirements/common-constraints.txt index 0fa6520cbdb..861b3a4cc9a 100644 --- a/requirements/common-constraints.txt +++ b/requirements/common-constraints.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --no-strip-extras --output-file=requirements/common-constraints.txt requirements/requirements.in requirements/requirements-dev.in requirements/requirements-help.in requirements/requirements-playwright.in +# uv pip compile --no-strip-extras --output-file=requirements/common-constraints.txt requirements/requirements.in requirements/requirements-dev.in requirements/requirements-help.in aiohappyeyeballs==2.6.1 # via aiohttp aiohttp==3.14.3 @@ -30,6 +30,10 @@ build==1.3.0 # via # chromadb # pip-tools +cdp-use==1.4.5 + # via cecli-browser-harness +cecli-browser-harness==0.1.13.post1 + # via -r requirements/requirements.in certifi==2025.11.12 # via # httpcore @@ -80,6 +84,8 @@ distlib==0.4.0 # via virtualenv durationpy==0.10 # via kubernetes +fetch-use==0.4.0 + # via cecli-browser-harness filelock==3.20.0 # via # huggingface-hub @@ -105,8 +111,6 @@ google-crc32c==1.8.0 # via moonshine-voice googleapis-common-protos==1.75.1 # via opentelemetry-exporter-otlp-proto-grpc -greenlet==3.2.4 - # via playwright grpcio==1.83.0 # via # chromadb @@ -125,6 +129,7 @@ httptools==0.8.0 # via uvicorn httpx==0.28.1 # via + # cdp-use # chromadb # mcp httpx-sse==0.4.3 @@ -265,9 +270,10 @@ pathspec==0.12.1 # via -r requirements/requirements.in pexpect==4.9.0 # via -r requirements/requirements.in -pillow==12.0.0 +pillow==12.3.0 # via # -r requirements/requirements.in + # cecli-browser-harness # matplotlib pip==25.3 # via pip-tools @@ -278,8 +284,6 @@ platformdirs==4.5.0 # moonshine-voice # textual # virtualenv -playwright==1.56.0 - # via -r requirements/requirements-playwright.in pluggy==1.6.0 # via pytest pre-commit==4.5.0 @@ -322,8 +326,6 @@ pydantic-settings==2.12.0 # mcp pydub==0.25.1 # via -r requirements/requirements.in -pyee==13.0.0 - # via playwright pyflakes==3.4.0 # via flake8 pygments==2.19.2 @@ -484,6 +486,7 @@ typing-extensions==4.15.0 # aiosignal # anyio # beautifulsoup4 + # cdp-use # chromadb # grpcio # huggingface-hub @@ -494,7 +497,6 @@ typing-extensions==4.15.0 # opentelemetry-semantic-conventions # pydantic # pydantic-core - # pyee # pytest-asyncio # referencing # starlette @@ -532,9 +534,11 @@ wcwidth==0.2.14 # via prompt-toolkit websocket-client==1.9.0 # via kubernetes -websockets==16.1.1 +websockets==17.4 # via # -r requirements/requirements.in + # cdp-use + # cecli-browser-harness # uvicorn wheel==0.45.1 # via pip-tools diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 7ecd3776847..3d2db719a9e 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -140,7 +140,7 @@ pandas==2.3.3 # via # -c requirements/common-constraints.txt # -r requirements/requirements-dev.in -pillow==12.0.0 +pillow==12.3.0 # via # -c requirements/common-constraints.txt # matplotlib diff --git a/requirements/requirements-help.txt b/requirements/requirements-help.txt index 5d25ac7ec34..07261be102a 100644 --- a/requirements/requirements-help.txt +++ b/requirements/requirements-help.txt @@ -368,7 +368,7 @@ websocket-client==1.9.0 # via # -c requirements/common-constraints.txt # kubernetes -websockets==16.1.1 +websockets==17.4 # via # -c requirements/common-constraints.txt # uvicorn diff --git a/requirements/requirements-playwright.in b/requirements/requirements-playwright.in deleted file mode 100644 index 508a5f469d2..00000000000 --- a/requirements/requirements-playwright.in +++ /dev/null @@ -1 +0,0 @@ -playwright diff --git a/requirements/requirements-playwright.txt b/requirements/requirements-playwright.txt deleted file mode 100644 index c3713486550..00000000000 --- a/requirements/requirements-playwright.txt +++ /dev/null @@ -1,18 +0,0 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile --no-strip-extras --constraint=requirements/common-constraints.txt --output-file=requirements/requirements-playwright.txt requirements/requirements-playwright.in --universal -greenlet==3.2.4 - # via - # -c requirements/common-constraints.txt - # playwright -playwright==1.56.0 - # via - # -c requirements/common-constraints.txt - # -r requirements/requirements-playwright.in -pyee==13.0.0 - # via - # -c requirements/common-constraints.txt - # playwright -typing-extensions==4.15.0 - # via - # -c requirements/common-constraints.txt - # pyee diff --git a/requirements/requirements.in b/requirements/requirements.in index bd251b356bb..c8e3b53cf9b 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -32,6 +32,11 @@ xxhash>=3.6.0 beautifulsoup4>=4.13.4 pypandoc>=1.15 +# browser automation and JavaScript-rendered scraping +# Fork widens browser-harness's websockets pin so cecli can stay on websockets>=16. +# Published to PyPI as cecli-browser-harness; PyPI rejects direct git URL dependencies. +cecli-browser-harness>=0.1.13.post1 ; python_version >= "3.11" + # storage/caching chromadb>=1.5.9 # chromadb's default ONNX embedder needs these directly; keep them in the base diff --git a/scripts/pip-compile.sh b/scripts/pip-compile.sh index 08e8c4f5890..245ad239d5d 100755 --- a/scripts/pip-compile.sh +++ b/scripts/pip-compile.sh @@ -30,7 +30,7 @@ grep -v ^tree-sitter= tmp.requirements.txt \ > requirements.txt # Compile additional requirements files -SUFFIXES=(dev help playwright) +SUFFIXES=(dev help) for SUFFIX in "${SUFFIXES[@]}"; do uv pip compile \ diff --git a/tests/basic/test_agent_config_merge.py b/tests/basic/test_agent_config_merge.py index 680dbb066b4..6bd02dd99e3 100644 --- a/tests/basic/test_agent_config_merge.py +++ b/tests/basic/test_agent_config_merge.py @@ -102,6 +102,7 @@ def test_all_yaml_to_json_args_deep_merge_with_config_file(): keys preserved, no underscore-key variants).""" assert set(YAML_TO_JSON_ARG_KEYS) == { "agent_config", + "browser", "tui_config", "mcp_servers", "custom", diff --git a/tests/helpers/test_key_detection.py b/tests/helpers/test_key_detection.py index 6a733cbef19..483222d2d6e 100644 --- a/tests/helpers/test_key_detection.py +++ b/tests/helpers/test_key_detection.py @@ -103,7 +103,7 @@ def test_ambiguous_suffix_names_require_key_like_values(): assert key_detection.detect_api_keys(merged, environ={}) == [] -def test_entropy_gate_rejects_whitespace_and_prose(): +def test_key_gate_rejects_whitespace_and_prose(): environ = {"sortKey": "rotate this api key every90 days"} assert key_detection.detect_api_keys(environ=environ) == [] @@ -123,10 +123,46 @@ def test_path_detection_handles_urls_windows_and_relative_paths(): assert not looks_like_path(FAKE_KEY) -def test_entropy_thresholds_cover_secret_words(): - assert set(key_detection.SECRET_NAME_SUFFIXES) <= set( - key_detection.CREDENTIAL_ENTROPY_THRESHOLDS - ) +def test_detects_provider_prefixes_for_ambiguous_names(): + environ = { + "sortKey": "AKIAABCDEFGHIJKLMNOPQRSTUV", + "lookupKey": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0", + } + + assert key_detection.detect_api_keys(environ=environ) == sorted(environ.values()) + + +def test_detects_short_same_case_prefixes_for_ambiguous_names(): + environ = { + "sortKey": "LLM_0123456789abcdefghijklmnopqrstuvwxyz", + "lookupKey": "abc_0123456789abcdefghijklmnopqrstuvwxyz", + } + + assert key_detection.detect_api_keys(environ=environ) == sorted(environ.values()) + + +def test_detects_mixed_case_and_alternating_values_for_ambiguous_names(): + environ = { + "sortKey": "aBcDeFgHiJkLmNoPqRsTuVwXyZ", + "lookupKey": "a2bc3d4ef5g6h7i8j9k0l1m2n3", + } + + assert key_detection.detect_api_keys(environ=environ) == sorted(environ.values()) + + +def test_rejects_path_starting_values_for_ambiguous_names(): + environ = { + "sortKey": "./aBcDeFgHiJkLmNoPqRsTuVwXyZ", + "lookupKey": r"C:\aBcDeFgHiJkLmNoPqRsTuVwXyZ", + } + + assert key_detection.detect_api_keys(environ=environ) == [] + + +def test_detects_private_and_jwt_named_values(): + environ = {"SIGNING_PRIVATE": "p@ss:w0rd!", "SESSION_JWT": "pa:ssw0rd"} + + assert set(key_detection.detect_api_keys(environ=environ)) == set(environ.values()) def test_keeps_slash_containing_secrets(): diff --git a/tests/helpers/test_loop_detect.py b/tests/helpers/test_loop_detect.py index 15cc7c5f99e..c5bf2cee41c 100644 --- a/tests/helpers/test_loop_detect.py +++ b/tests/helpers/test_loop_detect.py @@ -6,7 +6,8 @@ * what the detector can and cannot catch from LLM / small-language-model output, * the bounded-LRU behaviour of the sentence cache, -* the ``show_send_output_stream`` integration that stops streaming early, +* the ``show_send_output_stream`` integration that stops streaming early in agent + modes (single-shot modes skip loop tracking entirely), * the ``send``-level handling that marks the turn as ``[SYSTEM INTERJECTION: AVOID REPETITION]`` and drops any pending tool calls so they are never executed, * the adaptive per-level allowances that relax when a stream loops and bleed back @@ -227,7 +228,7 @@ def add_to_usage_report(self, *a, **k): return a[0] if a else "" -def _make_coder(): +def _make_coder(edit_format="agent"): coder = Coder.__new__(Coder) coder.stream = True coder.args = SimpleNamespace(debug=False, show_thinking=False) @@ -251,7 +252,7 @@ def _make_coder(): coder._output_loop_message = "" coder.repetition_penalties = LoopThresholdTracker() coder._has_empty_reflected = False - coder.edit_format = "code" + coder.edit_format = edit_format coder.max_compaction_retries = 3 coder.enable_context_compaction = False coder.model_kwargs = {} @@ -328,6 +329,24 @@ async def test_show_stream_detects_tool_args_loop_and_stops(): assert "Char loop" in coder._output_loop_message +@pytest.mark.parametrize("edit_format", ["code", "ask", "architect"]) +async def test_show_stream_skips_loop_detection_outside_agent_modes(edit_format): + coder = _make_coder(edit_format=edit_format) + chunks = [ + _content_chunk("a" * 50), + _content_chunk("a" * 51), + _tool_chunk([_tc(0, "call_1", "Local--ls", "a" * 101)]), + ] + coder.consolidate_chunks = lambda: (None, None, None) + + async for _ in coder.show_send_output_stream(_agen(chunks)): + pass + + assert coder._output_loop_detected is False + assert coder._output_loop_message == "" + assert coder.repetition_penalties.factors == {"char": 1.0, "word": 1.0, "sentence": 1.0} + + async def test_show_stream_does_not_flag_normal_stream(): coder = _make_coder() # Real consolidation is expensive here, so stub it; we only assert that a diff --git a/tests/scrape/test_playwright_disable.py b/tests/scrape/test_playwright_disable.py deleted file mode 100644 index 2d51f8a1f63..00000000000 --- a/tests/scrape/test_playwright_disable.py +++ /dev/null @@ -1,141 +0,0 @@ -from cecli.scrape import Scraper - - -class DummyIO: - def __init__(self): - self.outputs = [] - self.confirmed = False - - def tool_output(self, msg): - self.outputs.append(msg) - - def confirm_ask(self, msg, default="y"): - self.outputs.append(f"confirm: {msg}") - return self.confirmed - - def tool_error(self, msg): - self.outputs.append(f"error: {msg}") - - -async def test_scraper_disable_playwright_flag(monkeypatch): - io = DummyIO() - # Simulate that playwright is not available - # (disable_playwright just means playwright_available=False) - scraper = Scraper(print_error=io.tool_error, playwright_available=False) - # Patch scrape_with_httpx to check it is called - called = {} - - def fake_httpx(url): - called["called"] = True - return "plain text", "text/plain" - - scraper.scrape_with_httpx = fake_httpx - content = await scraper.scrape("http://example.com") - assert content == "plain text" - assert called["called"] - - -async def test_scraper_enable_playwright(monkeypatch): - io = DummyIO() - # Simulate that playwright is available and should be used - scraper = Scraper(print_error=io.tool_error, playwright_available=True) - # Patch scrape_with_playwright to check it is called - called = {} - - async def fake_playwright(url): - called["called"] = True - return "hi", "text/html" - - scraper.scrape_with_playwright = fake_playwright - content = await scraper.scrape("http://example.com") - assert content.startswith("hi") or "" in content - assert called["called"] - - -async def test_commands_web_disable_playwright(monkeypatch): - """ - Test that Commands.cmd_web does not emit a misleading warning when --disable-playwright is set. - """ - from cecli.commands import Commands - - # Dummy IO to capture outputs and warnings - class DummyIO: - def __init__(self): - self.outputs = [] - self.warnings = [] - self.errors = [] - - def tool_output(self, msg, *a, **k): - self.outputs.append(msg) - - def tool_warning(self, msg, *a, **k): - self.warnings.append(msg) - - def tool_error(self, msg, *a, **k): - self.errors.append(msg) - - def read_text(self, filename, silent=False): - return "" - - def confirm_ask(self, *a, **k): - return True - - def print(self, *a, **k): - pass - - # Dummy coder to satisfy Commands - class DummyCoder: - def __init__(self): - self.cur_messages = [] - self.main_model = type("M", (), {"edit_format": "code", "name": "dummy", "info": {}}) - self.args = type("Args", (), {"disable_playwright": True})() - self.io = io - self.tui = None - self.tui = None - - def get_rel_fname(self, fname): - return fname - - def get_inchat_relative_files(self): - return [] - - def abs_root_path(self, fname): - return fname - - def get_all_abs_files(self): - return [] - - def get_announcements(self): - return [] - - def format_chat_chunks(self): - return type("Chunks", (), {"repo": [], "readonly_files": [], "chat_files": []})() - - def event(self, *a, **k): - pass - - # Patch install_playwright to always return False (simulate not available) - - # Patch Scraper to always use scrape_with_httpx and never warn - class DummyScraper: - def __init__(self, **kwargs): - self.called = False - - async def scrape(self, url): - return "dummy content" - - monkeypatch.setattr("cecli.scrape.Scraper", DummyScraper) - - io = DummyIO() - coder = DummyCoder() - args = type("Args", (), {"disable_playwright": True})() - commands = Commands(io, coder, args=args) - - await commands.run("/web http://example.com") - # Should not emit a warning about playwright - assert not io.warnings - # Should not contain message "For the best web scraping, install Playwright:" - assert all("install Playwright:" not in msg for msg in io.outputs) - # Should output scraping and added to chat - assert any("Scraping" in msg for msg in io.outputs) - assert any("added to chat" in msg for msg in io.outputs) diff --git a/tests/scrape/test_scrape.py b/tests/scrape/test_scrape.py index 6a90ca18039..89b9dc8b5df 100644 --- a/tests/scrape/test_scrape.py +++ b/tests/scrape/test_scrape.py @@ -1,4 +1,4 @@ -import sys +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -20,9 +20,8 @@ def __init__(self): "M", (), {"edit_format": "code", "name": "dummy", "info": {}} )() self.tui = None - self.args = type("Args", (), {"disable_playwright": False})() + self.args = SimpleNamespace(browser=None) self.io = io - self.args = type("Args", (), {"disable_playwright": False})() def get_rel_fname(self, fname): return fname @@ -41,163 +40,96 @@ def get_announcements(self): coder = DummyCoder() cmds = Commands(io, coder) - cmds._test_coder = coder # keep strong reference to prevent GC of weakref + cmds._test_coder = coder return cmds - @patch("cecli.commands.web.install_playwright") + @patch("cecli.commands.web.is_browser_harness_available", return_value=True) @patch("cecli.commands.web.Scraper") - async def test_cmd_web_imports_playwright( - self, mock_scraper_class, mock_install_playwright, commands + async def test_cmd_web_uses_browser_harness_scraper( + self, mock_scraper_class, _mock_harness_available, commands ): - async def mock_install(*args, **kwargs): - sys.modules["playwright"] = MagicMock() - return True - - mock_install_playwright.side_effect = mock_install mock_scraper_instance = mock_scraper_class.return_value mock_scraper_instance.scrape = AsyncMock(return_value="Scraped content") - commands.io.tool_error = MagicMock() - try: - result = await commands.execute("web", "https://example.com", return_content=True) - - assert result is not None - assert "Scraped content" in result + result = await commands.execute("web", "https://example.com", return_content=True) - playwright_imported = "playwright" in sys.modules - assert playwright_imported, "Playwright should be importable after running cmd_web" + assert "Scraped content" in result + commands.io.tool_error.assert_not_called() - commands.io.tool_error.assert_not_called() - finally: - if "playwright" in sys.modules: - del sys.modules["playwright"] + _args, kwargs = mock_scraper_class.call_args + assert kwargs["browser_harness_available"] is True - @patch("cecli.scrape.Scraper.scrape_with_httpx") - @patch("cecli.scrape.Scraper.scrape_with_playwright") - async def test_scrape_self_signed_ssl(self, mock_scrape_playwright, mock_scrape_httpx): - # Test with SSL verification - playwright fails - mock_scrape_playwright.return_value = (None, None) - - scraper_verify = Scraper( - print_error=MagicMock(), playwright_available=True, verify_ssl=True - ) - result_verify = await scraper_verify.scrape("https://self-signed.badssl.com") - assert result_verify is None - scraper_verify.print_error.assert_called() - - # Test without SSL verification - playwright succeeds - mock_scrape_playwright.return_value = ( - "self-signed content", - "text/html", - ) - scraper_no_verify = Scraper( - print_error=MagicMock(), playwright_available=True, verify_ssl=False - ) - result_no_verify = await scraper_no_verify.scrape("https://self-signed.badssl.com") - assert result_no_verify is not None - assert "self-signed" in result_no_verify - scraper_no_verify.print_error.assert_not_called() - - @patch("cecli.scrape.Scraper.scrape_with_playwright") - async def test_scrape_actual_url_with_playwright(self, mock_scrape_playwright): - # Create a Scraper instance with a mock print_error function - mock_print_error = MagicMock() - scraper = Scraper(print_error=mock_print_error, playwright_available=True) - - # Mock the playwright scrape to return content - mock_scrape_playwright.return_value = ( - "

Example Domain

", - "text/html", + async def test_scrape_uses_browser_harness_when_available(self): + scraper = Scraper( + print_error=MagicMock(), + browser_harness_available=True, ) + html = "

Rendered

" + scraper.scrape_with_browser_harness = AsyncMock(return_value=(html, "text/html")) + scraper.html_to_markdown = MagicMock(return_value="# Rendered") - # Scrape a mocked URL result = await scraper.scrape("https://example.com") - # Assert that the result contains expected content - assert result is not None - assert "Example Domain" in result - - # Assert that print_error was never called - mock_print_error.assert_not_called() - - def test_scraper_print_error_not_called(self): - # Create a Scraper instance with a mock print_error function - mock_print_error = MagicMock() - scraper = Scraper(print_error=mock_print_error) - - # Test various methods of the Scraper class - scraper.scrape_with_httpx("https://example.com") - scraper.try_pandoc() - scraper.html_to_markdown("

Test

") + assert result == "# Rendered" + scraper.html_to_markdown.assert_called_once_with(html) - # Assert that print_error was never called - mock_print_error.assert_not_called() - - async def test_scrape_with_playwright_error_handling(self): - # Create a Scraper instance with a mock print_error function - mock_print_error = MagicMock() - scraper = Scraper(print_error=mock_print_error, playwright_available=True) - - # Mock the necessary objects and methods - scraper.scrape_with_playwright = AsyncMock() - scraper.scrape_with_playwright.return_value = (None, None) + async def test_scrape_falls_back_to_httpx_when_browser_harness_fails(self): + scraper = Scraper( + print_error=MagicMock(), + browser_harness_available=True, + ) + scraper.scrape_with_browser_harness = AsyncMock(return_value=(None, None)) + scraper.scrape_with_httpx = MagicMock(return_value=("plain text", "text/plain")) - # Call the scrape method result = await scraper.scrape("https://example.com") - # Assert that the result is None - assert result is None + assert result == "plain text" + scraper.scrape_with_httpx.assert_called_once_with("https://example.com") - # Assert that print_error was called with the expected error message - mock_print_error.assert_called_once_with( - "Failed to retrieve content from https://example.com" + async def test_scrape_without_browser_harness_uses_httpx(self): + scraper = Scraper( + print_error=MagicMock(), + browser_harness_available=False, ) + scraper.scrape_with_httpx = MagicMock(return_value=("plain text", "text/plain")) - # Reset the mock - mock_print_error.reset_mock() - - # Test with a different return value - scraper.scrape_with_playwright.return_value = ("Some content", "text/html") result = await scraper.scrape("https://example.com") - # Assert that the result is not None - assert result is not None - - # Assert that print_error was not called - mock_print_error.assert_not_called() + assert result == "plain text" - async def test_scrape_text_plain(self): - # Create a Scraper instance - scraper = Scraper(print_error=MagicMock(), playwright_available=True) - - # Mock the scrape_with_playwright method - plain_text = "This is plain text content." - scraper.scrape_with_playwright = AsyncMock(return_value=(plain_text, "text/plain")) + async def test_scrape_text_html(self): + scraper = Scraper( + print_error=MagicMock(), + browser_harness_available=True, + ) + html = "

Test

This is HTML content.

" + scraper.scrape_with_browser_harness = AsyncMock(return_value=(html, "text/html")) + scraper.html_to_markdown = MagicMock(return_value="# Test\n\nThis is HTML content.") - # Call the scrape method result = await scraper.scrape("https://example.com") - # Assert that the result is the same as the input plain text - assert result == plain_text + assert result == "# Test\n\nThis is HTML content." - async def test_scrape_text_html(self): - # Create a Scraper instance - scraper = Scraper(print_error=MagicMock(), playwright_available=True) + async def test_scrape_failure_returns_none(self): + scraper = Scraper( + print_error=MagicMock(), + browser_harness_available=False, + ) + scraper.scrape_with_httpx = MagicMock(return_value=(None, None)) - # Mock the scrape_with_playwright method - html_content = "

Test

This is HTML content.

" - scraper.scrape_with_playwright = AsyncMock(return_value=(html_content, "text/html")) + result = await scraper.scrape("https://example.com") - # Mock the html_to_markdown method - expected_markdown = "# Test\n\nThis is HTML content." - scraper.html_to_markdown = MagicMock(return_value=expected_markdown) + assert result is None + scraper.print_error.assert_called_once_with( + "Failed to retrieve content from https://example.com" + ) - # Call the scrape method - result = await scraper.scrape("https://example.com") + def test_scraper_print_error_not_called(self): + scraper = Scraper(print_error=MagicMock(), browser_harness_available=False) - # Assert that the result is the expected markdown - assert result == expected_markdown + scraper.scrape_with_httpx("https://example.com") + scraper.try_pandoc() + scraper.html_to_markdown("

Test

") - # Assert that html_to_markdown was called with the HTML content - scraper.html_to_markdown.assert_called_once_with(html_content) + scraper.print_error.assert_not_called() diff --git a/tests/subagents/test_service.py b/tests/subagents/test_service.py index 300d5932548..8a7aba969b7 100644 --- a/tests/subagents/test_service.py +++ b/tests/subagents/test_service.py @@ -570,6 +570,23 @@ async def test_unknown_name_raises(self, service): with pytest.raises(ValueError, match="Unknown sub-agent"): await service.spawn("ghost") + @pytest.mark.asyncio + async def test_spawn_builds_registry_when_empty(self, service): + """spawn loads built-in defaults for non-agent coders (issue #734).""" + assert AgentService.get_registry() == {} + + mock_new_coder = MagicMock() + mock_new_coder.tui = None + + with patch("cecli.coders.Coder") as MockCoder: + MockCoder.create = AsyncMock(return_value=mock_new_coder) + with patch("cecli.helpers.conversation.service.ConversationService") as MockConv: + MockConv.get_chunks.return_value = MagicMock() + + await service.spawn("creator") + + assert "creator" in AgentService.get_registry() + @pytest.mark.asyncio async def test_spawn_creates_without_generating(self, service, registry): """spawn creates sub-agent without calling generate.""" diff --git a/tests/tools/test_search_docs_addresses.py b/tests/tools/test_search_docs_addresses.py index 1ac17fd6168..9298e503985 100644 --- a/tests/tools/test_search_docs_addresses.py +++ b/tests/tools/test_search_docs_addresses.py @@ -141,7 +141,7 @@ async def test_toc_slugs_round_trip(): assert entry["_"]["sections"] == 13 assert entry["_"]["keys"] == 129 assert ' Merge Precedence (L28-183)" in entry["content"] + assert "merge-precedence YAML Config File > Merge Precedence (L28-184)" in entry["content"] follow_up = _result( await search_docs.Tool.execute(_coder(), path="config/conf.md#merge-precedence") @@ -149,7 +149,7 @@ async def test_toc_slugs_round_trip(): section = follow_up["result"][0] assert section["_"]["section"] == "YAML Config File > Merge Precedence" - assert section["_"]["range"] == [28, 183] + assert section["_"]["range"] == [28, 184] @pytest.mark.asyncio @@ -160,11 +160,11 @@ async def test_config_key_anchor_reads_the_key_pair(): assert entry["_"]["type"] == "doc_file" assert entry["_"]["key"] == "dark-mode" assert entry["_"]["value"] == "false" - assert entry["_"]["range"] == [361, 362] + assert entry["_"]["range"] == [362, 363] content = entry["content"] assert 'key="dark-mode"' in content - assert 'lines="361-362"' in content + assert 'lines="362-363"' in content assert "## Use colors suitable for a dark terminal background" in content assert "#dark-mode: false" in content diff --git a/tests/tools/test_search_docs_sections.py b/tests/tools/test_search_docs_sections.py index bd29c5b1cb7..83568e9191a 100644 --- a/tests/tools/test_search_docs_sections.py +++ b/tests/tools/test_search_docs_sections.py @@ -44,14 +44,14 @@ async def test_section_read_returns_only_that_subtree(): assert entry["_"]["section"] == CONF_SECTION assert entry["_"]["level"] == 2 - assert entry["_"]["range"] == [28, 183] - assert entry["_"]["lines"] == 646 + assert entry["_"]["range"] == [28, 184] + assert entry["_"]["lines"] == 650 assert content.startswith(f'