From 006db3989b414d896443f48c570d32043f9d2bc5 Mon Sep 17 00:00:00 2001 From: EmBista Date: Fri, 23 Oct 2026 13:31:53 +1100 Subject: [PATCH 1/2] Discover Codex models from latest stable release and account catalog Resolve the catalog client version from the latest stable openai/codex GitHub release, with an optional stable-only override and a bundled 0.160.0 fallback. Model listings come only from the account catalog and return 503 until one is available; priority support comes from catalog metadata. This replaces the hardcoded GPT-6 registry entries, priority list and cache invalidation from #126: an older-version cache is now kept and refreshed rather than discarded. Co-Authored-By: Claude Opus 5.5 (1M context) --- .env.example | 2 + DOCKER.md | 1 + README.md | 19 +++--- chatmock/fast_mode.py | 27 +-------- chatmock/model_catalog.py | 71 ++++++++++++++++++---- chatmock/model_registry.py | 38 ++---------- chatmock/routes_ollama.py | 2 + chatmock/routes_openai.py | 2 + tests/test_model_catalog.py | 114 +++++++++++++++++++++++++++++------- tests/test_routes.py | 56 +++++++++--------- 10 files changed, 203 insertions(+), 129 deletions(-) diff --git a/.env.example b/.env.example index f90573ae..043f5765 100644 --- a/.env.example +++ b/.env.example @@ -25,6 +25,8 @@ CHATGPT_LOCAL_EXPOSE_REASONING_MODELS=false # Automatically discover account models; refresh stale data without restarting CHATGPT_LOCAL_MODEL_SYNC=true CHATGPT_LOCAL_MODEL_REFRESH_INTERVAL=3600 +# Optional: override the latest stable Codex release used for model discovery +# CHATGPT_LOCAL_MODELS_CLIENT_VERSION=0.160.0 # Enable default web search tool CHATGPT_LOCAL_ENABLE_WEB_SEARCH=false diff --git a/DOCKER.md b/DOCKER.md index caf2b22f..b7322420 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -34,6 +34,7 @@ Set options in `.env` or pass environment variables: - `CHATGPT_LOCAL_ENABLE_WEB_SEARCH`: `true|false` to enable default web search tool - `CHATGPT_LOCAL_MODEL_SYNC`: `true|false` to discover account models automatically (default `true`) - `CHATGPT_LOCAL_MODEL_REFRESH_INTERVAL`: model catalog refresh interval in seconds (default `3600`) +- `CHATGPT_LOCAL_MODELS_CLIENT_VERSION`: optional stable Codex release version override; otherwise use the latest stable GitHub release, with a bundled fallback if GitHub is unavailable ## Logs Set `VERBOSE=true` to include extra logging for troubleshooting upstream or chat app requests. Please include and use these logs when submitting bug reports. diff --git a/README.md b/README.md index 9159478e..cc6994ed 100644 --- a/README.md +++ b/README.md @@ -85,18 +85,13 @@ The server runs at `http://127.0.0.1:8000` by default. Use `http://127.0.0.1:800 ## Supported Models ChatMock automatically discovers the models available to the signed-in ChatGPT -account. The current catalog commonly includes: - -- `gpt-6-astra` -- `gpt-6-sol` -- `gpt-6-luna` -- `gpt-5.6-sol` -- `gpt-5.6-terra` -- `gpt-5.6-luna` -- `gpt-5.5` -- `gpt-5.4` -- `gpt-5.4-mini` -- `gpt-5.3-codex-spark` +account. It checks the latest stable Codex release on GitHub when refreshing the +catalog and uses that version for the account's model request. An optional +`CHATGPT_LOCAL_MODELS_CLIENT_VERSION` override accepts a stable release +version only. If GitHub is unavailable, ChatMock uses the last successful version or +its bundled verified fallback. The model list and capabilities come from the +account catalog; `/v1/models` and `/api/tags` return 503 until one is available. +The catalog refresh interval defaults to one hour.
diff --git a/chatmock/fast_mode.py b/chatmock/fast_mode.py index a3cf4d26..3b01fc2e 100644 --- a/chatmock/fast_mode.py +++ b/chatmock/fast_mode.py @@ -5,24 +5,6 @@ from .model_registry import model_supports_service_tier, normalize_model_name - -PRIORITY_SUPPORTED_MODELS = frozenset( - ( - "gpt-5.4", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-5.2", - "gpt-5.1", - "gpt-5", - "gpt-5.1-codex", - "gpt-5-codex", - ) -) - _TRUE_STRINGS = {"1", "true", "yes", "on"} _FALSE_STRINGS = {"0", "false", "no", "off"} @@ -39,11 +21,8 @@ def parse_optional_bool(value: Any) -> bool | None: return None -def supports_priority_service_tier(model: str | None) -> bool: - catalog_support = model_supports_service_tier(model, "priority") - if catalog_support is not None: - return catalog_support - return normalize_model_name(model) in PRIORITY_SUPPORTED_MODELS +def supports_priority_service_tier(model: str | None) -> bool | None: + return model_supports_service_tier(model, "priority") @dataclass(frozen=True) @@ -77,7 +56,7 @@ def resolve_service_tier( tier = "priority" used_server_default = True - if tier == "priority" and not supports_priority_service_tier(model): + if tier == "priority" and supports_priority_service_tier(model) is False: normalized = normalize_model_name(model) message = ( f"Fast mode is not supported for model '{normalized}'. " diff --git a/chatmock/model_catalog.py b/chatmock/model_catalog.py index f57453ec..71ddb223 100644 --- a/chatmock/model_catalog.py +++ b/chatmock/model_catalog.py @@ -3,6 +3,7 @@ import datetime import json import os +import re import threading import time from dataclasses import dataclass @@ -24,9 +25,18 @@ DEFAULT_REFRESH_INTERVAL_SECONDS = 60 * 60 FAILED_REFRESH_RETRY_SECONDS = 60 FETCH_TIMEOUT_SECONDS = 5 +RELEASE_FETCH_TIMEOUT_SECONDS = 2 MODEL_CACHE_FILE = "chatmock_models_cache.json" -# Bump only after verifying ChatMock against a newer Codex catalog contract. CODEX_MODELS_CLIENT_VERSION = "0.160.0" +CODEX_RELEASE_URL = "https://api.github.com/repos/openai/codex/releases/latest" +CLIENT_VERSION_ENV = "CHATGPT_LOCAL_MODELS_CLIENT_VERSION" +CLIENT_VERSION_PATTERN = re.compile(r"^\d+\.\d+\.\d+$") +STABLE_RELEASE_TAG_PATTERN = re.compile(r"^rust-v(\d+\.\d+\.\d+)$") + + +def _version_parts(version: str) -> tuple[int, int, int]: + major, minor, patch = version.split("-", 1)[0].split(".") + return int(major), int(minor), int(patch) @dataclass(frozen=True) @@ -121,6 +131,11 @@ def __init__( self.refresh_interval_seconds = max(float(refresh_interval_seconds), 0.0) self.cache_path = Path(cache_path) if cache_path else Path(get_home_dir()) / MODEL_CACHE_FILE self._session = session or requests.Session() + override = (os.getenv(CLIENT_VERSION_ENV) or "").strip() + if override and not CLIENT_VERSION_PATTERN.fullmatch(override): + raise ValueError(f"{CLIENT_VERSION_ENV} must be a Codex version such as 0.160.0") + self._version_override = override or None + self._client_version = self._version_override or CODEX_MODELS_CLIENT_VERSION self._lock = threading.Lock() self._models: tuple[CatalogModel, ...] = () self._raw_models: list[dict[str, Any]] = [] @@ -171,7 +186,7 @@ def refresh_if_due(self, *, wait_for_refresh: bool = False) -> None: refresh_active = True if wait_for_refresh and refresh_active: - event.wait(FETCH_TIMEOUT_SECONDS + 1) + event.wait(RELEASE_FETCH_TIMEOUT_SECONDS + FETCH_TIMEOUT_SECONDS + 1) def _is_due_locked(self) -> bool: if not self._models or self._fetched_at is None: @@ -201,12 +216,13 @@ def _fetch_and_apply(self) -> None: if not access_token or not account_id: return - response = self._request_models(access_token, account_id) + client_version = self._resolve_client_version() + response = self._request_models(access_token, account_id, client_version) if response.status_code == 401: access_token, account_id = get_effective_chatgpt_auth(force_refresh=True) if not access_token or not account_id: return - response = self._request_models(access_token, account_id) + response = self._request_models(access_token, account_id, client_version) response.raise_for_status() payload = response.json() @@ -223,12 +239,36 @@ def _fetch_and_apply(self) -> None: self._fetched_at = fetched_at self._etag = etag self._account_id = account_id + self._client_version = client_version self._persist_cache() - def _request_models(self, access_token: str, account_id: str) -> requests.Response: + def _resolve_client_version(self) -> str: + if self._version_override: + return self._version_override + try: + response = self._session.get( + CODEX_RELEASE_URL, + headers={"Accept": "application/vnd.github+json"}, + timeout=RELEASE_FETCH_TIMEOUT_SECONDS, + ) + response.raise_for_status() + release = response.json() + if not isinstance(release, dict) or release.get("draft") or release.get("prerelease"): + return self._client_version + tag = release.get("tag_name") + match = STABLE_RELEASE_TAG_PATTERN.fullmatch(tag) if isinstance(tag, str) else None + if match and _version_parts(match.group(1)) > _version_parts(self._client_version): + return match.group(1) + except (requests.RequestException, ValueError, TypeError): + pass + return self._client_version + + def _request_models( + self, access_token: str, account_id: str, client_version: str + ) -> requests.Response: return self._session.get( f"{CHATGPT_CODEX_BASE_URL}/models", - params={"client_version": CODEX_MODELS_CLIENT_VERSION}, + params={"client_version": client_version}, headers={ "Authorization": f"Bearer {access_token}", "Accept": "application/json", @@ -248,8 +288,6 @@ def _load_cache(self) -> None: return if not isinstance(payload, dict): return - if payload.get("client_version") != CODEX_MODELS_CLIENT_VERSION: - return cached_account_id = payload.get("account_id") current_account_id = _account_id_from_auth_file() @@ -260,6 +298,15 @@ def _load_cache(self) -> None: ): return + cached_version = payload.get("client_version") + if ( + not self._version_override + and isinstance(cached_version, str) + and STABLE_RELEASE_TAG_PATTERN.fullmatch(f"rust-v{cached_version}") + and _version_parts(cached_version) > _version_parts(self._client_version) + ): + self._client_version = cached_version + raw_models = payload.get("models") parsed_models = _parse_models(raw_models) if not parsed_models or not any(model.visibility == "list" for model in parsed_models): @@ -267,7 +314,11 @@ def _load_cache(self) -> None: with self._lock: self._models = parsed_models self._raw_models = [dict(item) for item in raw_models if isinstance(item, dict)] - self._fetched_at = _parse_timestamp(payload.get("fetched_at")) + self._fetched_at = ( + _parse_timestamp(payload.get("fetched_at")) + if cached_version == self._client_version + else None + ) self._etag = payload.get("etag") if isinstance(payload.get("etag"), str) else None self._account_id = cached_account_id @@ -278,7 +329,7 @@ def _persist_cache(self) -> None: if self._fetched_at else None, "etag": self._etag, - "client_version": CODEX_MODELS_CLIENT_VERSION, + "client_version": self._client_version, "account_id": self._account_id, "models": self._raw_models, } diff --git a/chatmock/model_registry.py b/chatmock/model_registry.py index 25698e33..45b58341 100644 --- a/chatmock/model_registry.py +++ b/chatmock/model_registry.py @@ -83,27 +83,6 @@ class ModelSpec: allowed_efforts=frozenset(("low", "medium", "high", "xhigh", "max")), variant_efforts=("low", "medium", "high", "xhigh", "max"), ), - ModelSpec( - public_id="gpt-6-astra", - upstream_id="gpt-6-astra", - aliases=("gpt6-astra", "gpt-6-astra-latest"), - allowed_efforts=frozenset(("low", "medium", "high", "xhigh", "max", "ultra")), - variant_efforts=("low", "medium", "high", "xhigh", "max", "ultra"), - ), - ModelSpec( - public_id="gpt-6-sol", - upstream_id="gpt-6-sol", - aliases=("gpt6-sol", "gpt-6-sol-latest"), - allowed_efforts=frozenset(("low", "medium", "high", "xhigh", "max", "ultra")), - variant_efforts=("low", "medium", "high", "xhigh", "max", "ultra"), - ), - ModelSpec( - public_id="gpt-6-luna", - upstream_id="gpt-6-luna", - aliases=("gpt6-luna", "gpt-6-luna-latest"), - allowed_efforts=frozenset(("low", "medium", "high", "xhigh", "max")), - variant_efforts=("low", "medium", "high", "xhigh", "max"), - ), ModelSpec( public_id="gpt-5.3-codex", upstream_id="gpt-5.3-codex", @@ -267,21 +246,12 @@ def extract_reasoning_from_model_name(model: str | None) -> dict[str, str] | Non def list_public_models(expose_reasoning_models: bool = False) -> list[str]: catalog = current_model_catalog() - if catalog is not None: - remote_models = catalog.visible_models(wait_for_refresh=True) - if remote_models: - model_ids: list[str] = [] - for model in remote_models: - model_ids.append(model.slug) - if expose_reasoning_models: - model_ids.extend(f"{model.slug}-{effort}" for effort in model.reasoning_efforts) - return model_ids - + remote_models = catalog.visible_models(wait_for_refresh=True) if catalog is not None else () model_ids: list[str] = [] - for spec in _MODEL_SPECS: - model_ids.append(spec.public_id) + for model in remote_models: + model_ids.append(model.slug) if expose_reasoning_models: - model_ids.extend(f"{spec.public_id}-{effort}" for effort in spec.variant_efforts) + model_ids.extend(f"{model.slug}-{effort}" for effort in model.reasoning_efforts) return model_ids diff --git a/chatmock/routes_ollama.py b/chatmock/routes_ollama.py index 60480112..a30ee61d 100644 --- a/chatmock/routes_ollama.py +++ b/chatmock/routes_ollama.py @@ -95,6 +95,8 @@ def ollama_tags() -> Response: print("IN GET /api/tags") expose_variants = bool(current_app.config.get("EXPOSE_REASONING_MODELS")) model_ids = list_public_models(expose_reasoning_models=expose_variants) + if not model_ids: + return make_response(jsonify({"error": "Model catalog unavailable"}), 503) models = [] for model_id in model_ids: models.append( diff --git a/chatmock/routes_openai.py b/chatmock/routes_openai.py index 882d34c3..f9e939a3 100644 --- a/chatmock/routes_openai.py +++ b/chatmock/routes_openai.py @@ -715,6 +715,8 @@ def responses_create() -> Response: def list_models() -> Response: expose_variants = bool(current_app.config.get("EXPOSE_REASONING_MODELS")) model_ids = list_public_models(expose_reasoning_models=expose_variants) + if not model_ids: + return make_response(jsonify({"error": {"message": "Model catalog unavailable"}}), 503) data = [{"id": mid, "object": "model", "owned_by": "owner"} for mid in model_ids] models = {"object": "list", "data": data} resp = make_response(jsonify(models), 200) diff --git a/tests/test_model_catalog.py b/tests/test_model_catalog.py index 69d86e7f..4484a998 100644 --- a/tests/test_model_catalog.py +++ b/tests/test_model_catalog.py @@ -4,32 +4,106 @@ import tempfile import unittest from pathlib import Path -from unittest.mock import patch +from unittest.mock import Mock, patch -from chatmock.model_catalog import CODEX_MODELS_CLIENT_VERSION, ModelCatalog +import requests +from chatmock.model_catalog import ( + CLIENT_VERSION_ENV, + CODEX_MODELS_CLIENT_VERSION, + CODEX_RELEASE_URL, + CatalogModel, + ModelCatalog, +) +from chatmock.app import create_app +from chatmock.fast_mode import supports_priority_service_tier -class ModelCatalogCacheTests(unittest.TestCase): - def test_cache_from_older_client_version_is_ignored(self) -> None: + +def response(payload: dict, status: int = 200) -> Mock: + result = Mock() + result.status_code = status + result.json.return_value = payload + result.headers = {} + result.raise_for_status.side_effect = ( + requests.HTTPError("release unavailable") if status >= 400 else None + ) + return result + + +class ModelCatalogTests(unittest.TestCase): + def test_latest_stable_release_supplies_catalog_client_version(self) -> None: + session = Mock() + session.get.side_effect = [ + response({"tag_name": "rust-v9.0.0", "prerelease": False, "draft": False}), + response({"models": [{"slug": "future-model", "visibility": "list"}]}), + ] with tempfile.TemporaryDirectory() as directory: - cache_path = Path(directory) / "models.json" - payload = { - "account_id": "test-account", - "client_version": "0.146.0", - "fetched_at": "2026-09-22T00:00:00Z", - "models": [{"slug": "gpt-5.6-sol", "visibility": "list"}], - } - cache_path.write_text(json.dumps(payload), encoding="utf-8") + catalog = ModelCatalog( + enabled=False, session=session, cache_path=Path(directory) / "models.json" + ) + with patch("chatmock.model_catalog.get_effective_chatgpt_auth", return_value=("token", "account")): + catalog._fetch_and_apply() + + self.assertEqual(session.get.call_args_list[0].args[0], CODEX_RELEASE_URL) + self.assertEqual(session.get.call_args_list[1].kwargs["params"], {"client_version": "9.0.0"}) + self.assertEqual([model.slug for model in catalog.visible_models()], ["future-model"]) - with patch("chatmock.model_catalog._account_id_from_auth_file", return_value="test-account"): - old_catalog = ModelCatalog(cache_path=cache_path) - self.assertEqual(old_catalog._models, ()) + def test_invalid_release_or_github_failure_uses_verified_fallback(self) -> None: + for release in ( + response({"tag_name": "rust-v9.0.0-alpha.9", "prerelease": True}), + response({"tag_name": "unrelated-v2.0.0", "prerelease": False}), + response({}, 503), + ): + with self.subTest(release=release): + session = Mock() + session.get.return_value = release + catalog = ModelCatalog(enabled=False, session=session) + self.assertEqual(catalog._resolve_client_version(), CODEX_MODELS_CLIENT_VERSION) + + def test_environment_override_skips_github(self) -> None: + session = Mock() + with patch.dict("os.environ", {CLIENT_VERSION_ENV: "0.155.0"}): + catalog = ModelCatalog(enabled=False, session=session) + self.assertEqual(catalog._resolve_client_version(), "0.155.0") + session.get.assert_not_called() + + def test_invalid_environment_override_fails_loudly(self) -> None: + for version in ("latest", "9.0.0-alpha.1"): + with self.subTest(version=version): + with patch.dict("os.environ", {CLIENT_VERSION_ENV: version}): + with self.assertRaises(ValueError): + ModelCatalog(enabled=False) + + def test_cache_from_older_version_is_refreshed_without_losing_last_models(self) -> None: + with tempfile.TemporaryDirectory() as directory: + cache_path = Path(directory) / "models.json" + cache_path.write_text( + json.dumps({ + "account_id": "account", + "client_version": "0.146.0", + "fetched_at": "2026-09-23T00:00:00Z", + "models": [{"slug": "old-model", "visibility": "list"}], + }), + encoding="utf-8", + ) + with patch("chatmock.model_catalog._account_id_from_auth_file", return_value="account"): + catalog = ModelCatalog(cache_path=cache_path) + self.assertEqual([model.slug for model in catalog._models], ["old-model"]) + self.assertIsNone(catalog._fetched_at) + self.assertEqual(catalog._client_version, CODEX_MODELS_CLIENT_VERSION) - payload["client_version"] = CODEX_MODELS_CLIENT_VERSION - cache_path.write_text(json.dumps(payload), encoding="utf-8") - current_catalog = ModelCatalog(cache_path=cache_path) - self.assertEqual([model.slug for model in current_catalog._models], ["gpt-5.6-sol"]) + def test_future_model_priority_uses_catalog_metadata(self) -> None: + app = create_app(model_sync=False) + catalog = app.extensions["chatmock_model_catalog"] + catalog._models = ( + CatalogModel("future-model", ("low",), frozenset(("priority",)), 1, "list", True), + CatalogModel("ordinary-model", ("low",), frozenset(), 2, "list", True), + ) + with app.app_context(): + self.assertTrue(supports_priority_service_tier("future-model")) + self.assertFalse(supports_priority_service_tier("ordinary-model")) + self.assertIsNone(supports_priority_service_tier("unlisted-model")) if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/test_routes.py b/tests/test_routes.py index 21b82c93..3fa114c2 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -8,6 +8,7 @@ from unittest.mock import patch from chatmock.app import create_app +from chatmock.model_catalog import CatalogModel from chatmock.session import reset_session_state from websockets.sync.client import connect as ws_connect @@ -54,32 +55,25 @@ def setUp(self) -> None: self.app = create_app(model_sync=False) self.client = self.app.test_client() - def test_openai_models_list(self) -> None: + def test_model_lists_require_a_discovered_catalog(self) -> None: response = self.client.get("/v1/models") - body = response.get_json() - self.assertEqual(response.status_code, 200) - model_ids = [item["id"] for item in body["data"]] - self.assertIn("gpt-5.4", model_ids) - self.assertIn("gpt-5.4-mini", model_ids) - self.assertIn("gpt-5.3-codex-spark", model_ids) - self.assertIn("gpt-5.6-sol", model_ids) - self.assertIn("gpt-5.6-terra", model_ids) - self.assertIn("gpt-5.6-luna", model_ids) - for model in ("gpt-6-astra", "gpt-6-sol", "gpt-6-luna"): - self.assertIn(model, model_ids) - - def test_ollama_tags_list(self) -> None: - response = self.client.get("/api/tags") - body = response.get_json() - self.assertEqual(response.status_code, 200) - model_names = [item["name"] for item in body["models"]] - self.assertIn("gpt-5.4", model_names) - self.assertIn("gpt-5.4-mini", model_names) - self.assertIn("gpt-5.6-sol", model_names) - self.assertIn("gpt-5.6-terra", model_names) - self.assertIn("gpt-5.6-luna", model_names) - for model in ("gpt-6-astra", "gpt-6-sol", "gpt-6-luna"): - self.assertIn(model, model_names) + self.assertEqual(response.status_code, 503) + self.assertEqual(self.client.get("/api/tags").status_code, 503) + + @patch("chatmock.model_catalog.ModelCatalog.visible_models") + def test_model_lists_use_discovered_models_without_registration(self, mock_visible) -> None: + mock_visible.return_value = ( + CatalogModel("future-model", ("low", "high"), frozenset(("priority",)), 1, "list", True), + ) + app = create_app(model_sync=True, expose_reasoning_models=True) + client = app.test_client() + openai = client.get("/v1/models") + ollama = client.get("/api/tags") + self.assertEqual(openai.status_code, 200) + self.assertEqual(ollama.status_code, 200) + expected = ["future-model", "future-model-low", "future-model-high"] + self.assertEqual([item["id"] for item in openai.get_json()["data"]], expected) + self.assertEqual([item["name"] for item in ollama.get_json()["models"]], expected) @patch("chatmock.routes_openai.start_upstream_request") def test_chat_completions(self, mock_start) -> None: @@ -163,8 +157,9 @@ def test_ollama_chat_honors_debug_model_override(self, mock_start) -> None: self.assertEqual(mock_start.call_args.args[0], "gpt-5.4") self.assertEqual(body["model"], "gpt-5.4") + @patch("chatmock.fast_mode.model_supports_service_tier", return_value=True) @patch("chatmock.routes_openai.start_upstream_request") - def test_chat_completions_fast_mode_sets_priority_service_tier(self, mock_start) -> None: + def test_chat_completions_fast_mode_sets_priority_service_tier(self, mock_start, _mock_support) -> None: mock_start.return_value = ( FakeUpstream( [ @@ -209,8 +204,9 @@ def test_chat_completions_fast_mode_false_overrides_server_default(self, mock_st self.assertEqual(response.status_code, 200) self.assertIsNone(mock_start.call_args.kwargs["service_tier"]) + @patch("chatmock.fast_mode.model_supports_service_tier", return_value=False) @patch("chatmock.routes_openai.start_upstream_request") - def test_chat_completions_rejects_unsupported_explicit_fast_mode(self, mock_start) -> None: + def test_chat_completions_rejects_unsupported_explicit_fast_mode(self, mock_start, _mock_support) -> None: response = self.client.post( "/v1/chat/completions", json={ @@ -657,8 +653,9 @@ def test_responses_route_stream_passthrough(self, mock_start) -> None: self.assertEqual(response.status_code, 200) self.assertIn("response.output_text.delta", response.get_data(as_text=True)) + @patch("chatmock.fast_mode.model_supports_service_tier", return_value=False) @patch("chatmock.routes_openai.start_upstream_raw_request") - def test_responses_route_rejects_unsupported_explicit_priority(self, mock_start) -> None: + def test_responses_route_rejects_unsupported_explicit_priority(self, mock_start, _mock_support) -> None: response = self.client.post( "/v1/responses", json={"model": "gpt-5.3-codex", "input": "hello", "service_tier": "priority"}, @@ -668,9 +665,10 @@ def test_responses_route_rejects_unsupported_explicit_priority(self, mock_start) self.assertIn("Fast mode is not supported", body["error"]["message"]) mock_start.assert_not_called() + @patch("chatmock.fast_mode.model_supports_service_tier", return_value=True) @patch("chatmock.websocket_routes.get_effective_chatgpt_auth", return_value=("token", "acct")) @patch("chatmock.websocket_routes.connect_upstream_websocket") - def test_responses_websocket_rewrites_response_create(self, mock_connect, _mock_auth) -> None: + def test_responses_websocket_rewrites_response_create(self, mock_connect, _mock_auth, _mock_support) -> None: class FakeUpstreamWebsocket: def __init__(self) -> None: self.sent: list[str] = [] From 2dbc45f4e2166417528b8d8a2918f15fb3592071 Mon Sep 17 00:00:00 2001 From: EmBista Date: Fri, 23 Oct 2026 13:35:58 +1100 Subject: [PATCH 2/2] Use the account catalog for model metadata and the request default - /api/show reports the catalog's context window and input modalities instead of a fixed 2,000,000-token context and vision for every model. Models the catalog does not know keep the previous response. - A request without a model uses the catalog's top visible model; gpt-5.4 remains the default only when no catalog is available. - Share one catalog lookup between /api/show and priority-tier checks. - Take the CLI --reasoning-effort choices from the registry's effort list. - Remove the unused iter_public_models(). Co-Authored-By: Claude Opus 5.5 (1M context) --- chatmock/cli.py | 3 ++- chatmock/model_catalog.py | 14 ++++++++++++++ chatmock/model_registry.py | 25 +++++++++++++++---------- chatmock/routes_ollama.py | 20 ++++++++++++++++---- tests/test_model_catalog.py | 33 +++++++++++++++++++++++++++++++++ tests/test_routes.py | 13 +++++++++++++ 6 files changed, 93 insertions(+), 15 deletions(-) diff --git a/chatmock/cli.py b/chatmock/cli.py index 1383bf92..f99ecc65 100644 --- a/chatmock/cli.py +++ b/chatmock/cli.py @@ -11,6 +11,7 @@ from .app import create_app from .config import CLIENT_ID_DEFAULT from .limits import RateLimitWindow, compute_reset_at, load_rate_limit_snapshot +from .model_registry import ALL_REASONING_EFFORTS from .oauth import OAuthHTTPServer, OAuthHandler, REQUIRED_PORT, URL_BASE, run_device_code_login from .utils import eprint, get_home_dir, load_chatgpt_tokens, parse_jwt_claims, read_auth_file @@ -293,7 +294,7 @@ def main() -> None: ) p_serve.add_argument( "--reasoning-effort", - choices=["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"], + choices=list(ALL_REASONING_EFFORTS), default=os.getenv("CHATGPT_LOCAL_REASONING_EFFORT", "medium").lower(), help="Reasoning effort level for Responses API (default: medium)", ) diff --git a/chatmock/model_catalog.py b/chatmock/model_catalog.py index 71ddb223..830fb46c 100644 --- a/chatmock/model_catalog.py +++ b/chatmock/model_catalog.py @@ -47,6 +47,8 @@ class CatalogModel: priority: int visibility: str supported_in_api: bool + context_window: int | None = None + input_modalities: tuple[str, ...] = () def _now_utc() -> datetime.datetime: @@ -102,7 +104,15 @@ def _parse_models(value: Any) -> tuple[CatalogModel, ...]: if isinstance(tier, str) and tier.strip(): service_tiers.add(tier.strip()) + modalities: list[str] = [] + raw_modalities = item.get("input_modalities") + if isinstance(raw_modalities, list): + for modality in raw_modalities: + if isinstance(modality, str) and modality.strip() and modality.strip() not in modalities: + modalities.append(modality.strip()) + priority = item.get("priority") + context_window = item.get("context_window") models.append( CatalogModel( slug=slug.strip(), @@ -111,6 +121,10 @@ def _parse_models(value: Any) -> tuple[CatalogModel, ...]: priority=priority if isinstance(priority, int) else 1_000_000, visibility=item.get("visibility") if isinstance(item.get("visibility"), str) else "none", supported_in_api=bool(item.get("supported_in_api", False)), + context_window=context_window + if isinstance(context_window, int) and not isinstance(context_window, bool) and context_window > 0 + else None, + input_modalities=tuple(modalities), ) ) return tuple(models) diff --git a/chatmock/model_registry.py b/chatmock/model_registry.py index 45b58341..8545129c 100644 --- a/chatmock/model_registry.py +++ b/chatmock/model_registry.py @@ -1,11 +1,11 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Iterable - from .model_catalog import CatalogModel, current_model_catalog +# Request default when no model is given and no account catalog is available. +FALLBACK_DEFAULT_MODEL = "gpt-5.4" ALL_REASONING_EFFORTS = ("none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra") DEFAULT_REASONING_EFFORTS = frozenset(ALL_REASONING_EFFORTS) @@ -224,7 +224,9 @@ def normalize_model_name(model: str | None, debug_model: str | None = None) -> s return spec.upstream_id if isinstance(model, str) and model.strip(): return model.strip() - return "gpt-5.4" + catalog = current_model_catalog() + visible = catalog.visible_models() if catalog is not None else () + return visible[0].slug if visible else FALLBACK_DEFAULT_MODEL def allowed_efforts_for_model(model: str | None) -> frozenset[str]: @@ -255,15 +257,18 @@ def list_public_models(expose_reasoning_models: bool = False) -> list[str]: return model_ids -def iter_public_models() -> Iterable[ModelSpec]: - return _MODEL_SPECS - - -def model_supports_service_tier(model: str | None, service_tier: str) -> bool | None: +def catalog_model_for_name(model: str | None) -> CatalogModel | None: spec, _ = _resolve_remote_model(model) if spec is None: return None for remote_model in _remote_models(): if remote_model.slug == spec.upstream_id: - return service_tier in remote_model.service_tiers - return False + return remote_model + return None + + +def model_supports_service_tier(model: str | None, service_tier: str) -> bool | None: + remote_model = catalog_model_for_name(model) + if remote_model is None: + return None + return service_tier in remote_model.service_tiers diff --git a/chatmock/routes_ollama.py b/chatmock/routes_ollama.py index a30ee61d..3b7303f3 100644 --- a/chatmock/routes_ollama.py +++ b/chatmock/routes_ollama.py @@ -10,7 +10,7 @@ from .fast_mode import resolve_service_tier from .limits import record_rate_limits_from_response from .http import build_cors_headers -from .model_registry import list_public_models +from .model_registry import catalog_model_for_name, list_public_models from .reasoning import ( allowed_efforts_for_model, build_reasoning_param, @@ -144,8 +144,20 @@ def ollama_show() -> Response: if verbose: _log_json("OUT POST /api/show", err) return jsonify(err), 400 + context_length = 2000000 + num_ctx = 100000 + capabilities = ["completion", "vision", "tools", "thinking"] + catalog_model = catalog_model_for_name(model) + if catalog_model is not None: + if catalog_model.context_window: + context_length = num_ctx = catalog_model.context_window + capabilities = ["completion", "tools"] + if "image" in catalog_model.input_modalities: + capabilities.append("vision") + if catalog_model.reasoning_efforts: + capabilities.append("thinking") v1_show_response = { - "modelfile": "# Modelfile generated by \"ollama show\"\n# To build a new Modelfile based on this one, replace the FROM line with:\n# FROM llava:latest\n\nFROM /models/blobs/sha256:placeholder\nTEMPLATE \"\"\"{{ .System }}\nUSER: {{ .Prompt }}\nASSISTANT: \"\"\"\nPARAMETER num_ctx 100000\nPARAMETER stop \"\"\nPARAMETER stop \"USER:\"\nPARAMETER stop \"ASSISTANT:\"", + "modelfile": "# Modelfile generated by \"ollama show\"\n# To build a new Modelfile based on this one, replace the FROM line with:\n# FROM llava:latest\n\nFROM /models/blobs/sha256:placeholder\nTEMPLATE \"\"\"{{ .System }}\nUSER: {{ .Prompt }}\nASSISTANT: \"\"\"\nPARAMETER num_ctx " + str(num_ctx) + "\nPARAMETER stop \"\"\nPARAMETER stop \"USER:\"\nPARAMETER stop \"ASSISTANT:\"", "parameters": "num_keep 24\nstop \"<|start_header_id|>\"\nstop \"<|end_header_id|>\"\nstop \"<|eot_id|>\"", "template": "{{ if .System }}<|start_header_id|>system<|end_header_id|>\n\n{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>\n\n{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>\n\n{{ .Response }}<|eot_id|>", "details": { @@ -159,9 +171,9 @@ def ollama_show() -> Response: "model_info": { "general.architecture": "llama", "general.file_type": 2, - "llama.context_length": 2000000, + "llama.context_length": context_length, }, - "capabilities": ["completion", "vision", "tools", "thinking"], + "capabilities": capabilities, } if verbose: _log_json("OUT POST /api/show", v1_show_response) diff --git a/tests/test_model_catalog.py b/tests/test_model_catalog.py index 4484a998..d4d7838f 100644 --- a/tests/test_model_catalog.py +++ b/tests/test_model_catalog.py @@ -17,6 +17,7 @@ ) from chatmock.app import create_app from chatmock.fast_mode import supports_priority_service_tier +from chatmock.model_registry import FALLBACK_DEFAULT_MODEL, normalize_model_name def response(payload: dict, status: int = 200) -> Mock: @@ -104,6 +105,38 @@ def test_future_model_priority_uses_catalog_metadata(self) -> None: self.assertFalse(supports_priority_service_tier("ordinary-model")) self.assertIsNone(supports_priority_service_tier("unlisted-model")) + def test_catalog_metadata_is_parsed(self) -> None: + session = Mock() + session.get.side_effect = [ + response({}, 503), + response({"models": [{ + "slug": "future-model", + "visibility": "list", + "context_window": 272000, + "input_modalities": ["text", "image"], + }]}), + ] + with tempfile.TemporaryDirectory() as directory: + catalog = ModelCatalog( + enabled=False, session=session, cache_path=Path(directory) / "models.json" + ) + with patch("chatmock.model_catalog.get_effective_chatgpt_auth", return_value=("token", "account")): + catalog._fetch_and_apply() + model = catalog.visible_models()[0] + self.assertEqual(model.context_window, 272000) + self.assertEqual(model.input_modalities, ("text", "image")) + + def test_default_model_comes_from_catalog(self) -> None: + app = create_app(model_sync=False) + with app.app_context(): + self.assertEqual(normalize_model_name(None), FALLBACK_DEFAULT_MODEL) + app.extensions["chatmock_model_catalog"]._models = ( + CatalogModel("second-model", ("low",), frozenset(), 2, "list", True), + CatalogModel("hidden-model", ("low",), frozenset(), 0, "hide", True), + CatalogModel("first-model", ("low",), frozenset(), 1, "list", True), + ) + self.assertEqual(normalize_model_name(None), "first-model") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_routes.py b/tests/test_routes.py index 3fa114c2..2e99a865 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -75,6 +75,19 @@ def test_model_lists_use_discovered_models_without_registration(self, mock_visib self.assertEqual([item["id"] for item in openai.get_json()["data"]], expected) self.assertEqual([item["name"] for item in ollama.get_json()["models"]], expected) + def test_ollama_show_uses_catalog_metadata(self) -> None: + self.app.extensions["chatmock_model_catalog"]._models = ( + CatalogModel("future-model", ("low",), frozenset(), 1, "list", True, 272000, ("text", "image")), + CatalogModel("text-model", (), frozenset(), 2, "list", True, 128000, ("text",)), + ) + future = self.client.post("/api/show", json={"model": "future-model"}).get_json() + self.assertEqual(future["model_info"]["llama.context_length"], 272000) + self.assertIn("PARAMETER num_ctx 272000", future["modelfile"]) + self.assertEqual(future["capabilities"], ["completion", "tools", "vision", "thinking"]) + text = self.client.post("/api/show", json={"model": "text-model"}).get_json() + self.assertEqual(text["model_info"]["llama.context_length"], 128000) + self.assertEqual(text["capabilities"], ["completion", "tools"]) + @patch("chatmock.routes_openai.start_upstream_request") def test_chat_completions(self, mock_start) -> None: mock_start.return_value = (