diff --git a/.env.example b/.env.example index f90573ae..e0891c16 100644 --- a/.env.example +++ b/.env.example @@ -25,6 +25,8 @@ CHATGPT_LOCAL_EXPOSE_REASONING_MODELS=false # Automatically discover account models; refresh stale data without restarting CHATGPT_LOCAL_MODEL_SYNC=true CHATGPT_LOCAL_MODEL_REFRESH_INTERVAL=3600 +# Optional: override the latest stable Codex release used for model discovery +# CHATGPT_LOCAL_MODELS_CLIENT_VERSION=0.156.0 # Enable default web search tool CHATGPT_LOCAL_ENABLE_WEB_SEARCH=false diff --git a/DOCKER.md b/DOCKER.md index caf2b22f..b7322420 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -34,6 +34,7 @@ Set options in `.env` or pass environment variables: - `CHATGPT_LOCAL_ENABLE_WEB_SEARCH`: `true|false` to enable default web search tool - `CHATGPT_LOCAL_MODEL_SYNC`: `true|false` to discover account models automatically (default `true`) - `CHATGPT_LOCAL_MODEL_REFRESH_INTERVAL`: model catalog refresh interval in seconds (default `3600`) +- `CHATGPT_LOCAL_MODELS_CLIENT_VERSION`: optional stable Codex release version override; otherwise use the latest stable GitHub release, with a bundled fallback if GitHub is unavailable ## Logs Set `VERBOSE=true` to include extra logging for troubleshooting upstream or chat app requests. Please include and use these logs when submitting bug reports. diff --git a/README.md b/README.md index 0df3503a..89bc9ca5 100644 --- a/README.md +++ b/README.md @@ -88,15 +88,13 @@ The server runs at `http://127.0.0.1:8000` by default. Use `http://127.0.0.1:800 ## Supported Models ChatMock automatically discovers the models available to the signed-in ChatGPT -account. The current catalog commonly includes: - -- `gpt-5.6-sol` -- `gpt-5.6-terra` -- `gpt-5.6-luna` -- `gpt-5.5` -- `gpt-5.4` -- `gpt-5.4-mini` -- `gpt-5.3-codex-spark` +account. It checks the latest stable Codex release on GitHub when refreshing the +catalog and uses that version for the account's model request. An optional +`CHATGPT_LOCAL_MODELS_CLIENT_VERSION` override accepts a stable release +version only. If GitHub is unavailable, ChatMock uses the last successful version or +its bundled verified fallback. The model list and capabilities come from the +account catalog; `/v1/models` and `/api/tags` return 503 until one is available. +The catalog refresh interval defaults to one hour.
diff --git a/chatmock/fast_mode.py b/chatmock/fast_mode.py index 498b70b8..3b01fc2e 100644 --- a/chatmock/fast_mode.py +++ b/chatmock/fast_mode.py @@ -5,21 +5,6 @@ from .model_registry import model_supports_service_tier, normalize_model_name - -PRIORITY_SUPPORTED_MODELS = frozenset( - ( - "gpt-5.4", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.2", - "gpt-5.1", - "gpt-5", - "gpt-5.1-codex", - "gpt-5-codex", - ) -) - _TRUE_STRINGS = {"1", "true", "yes", "on"} _FALSE_STRINGS = {"0", "false", "no", "off"} @@ -36,11 +21,8 @@ def parse_optional_bool(value: Any) -> bool | None: return None -def supports_priority_service_tier(model: str | None) -> bool: - catalog_support = model_supports_service_tier(model, "priority") - if catalog_support is not None: - return catalog_support - return normalize_model_name(model) in PRIORITY_SUPPORTED_MODELS +def supports_priority_service_tier(model: str | None) -> bool | None: + return model_supports_service_tier(model, "priority") @dataclass(frozen=True) @@ -74,11 +56,11 @@ def resolve_service_tier( tier = "priority" used_server_default = True - if tier == "priority" and not supports_priority_service_tier(model): + if tier == "priority" and supports_priority_service_tier(model) is False: normalized = normalize_model_name(model) message = ( f"Fast mode is not supported for model '{normalized}'. " - "Use a supported GPT-5 priority-processing model or disable fast mode for this request." + "Use a model with priority processing or disable fast mode for this request." ) if explicit_request: return ServiceTierResolution( diff --git a/chatmock/model_catalog.py b/chatmock/model_catalog.py index 89459608..05b5459e 100644 --- a/chatmock/model_catalog.py +++ b/chatmock/model_catalog.py @@ -3,6 +3,7 @@ import datetime import json import os +import re import threading import time from dataclasses import dataclass @@ -24,9 +25,18 @@ DEFAULT_REFRESH_INTERVAL_SECONDS = 60 * 60 FAILED_REFRESH_RETRY_SECONDS = 60 FETCH_TIMEOUT_SECONDS = 5 +RELEASE_FETCH_TIMEOUT_SECONDS = 2 MODEL_CACHE_FILE = "chatmock_models_cache.json" -# Bump only after verifying ChatMock against a newer Codex catalog contract. -CODEX_MODELS_CLIENT_VERSION = "0.146.0" +CODEX_MODELS_CLIENT_VERSION = "0.156.0" +CODEX_RELEASE_URL = "https://api.github.com/repos/openai/codex/releases/latest" +CLIENT_VERSION_ENV = "CHATGPT_LOCAL_MODELS_CLIENT_VERSION" +CLIENT_VERSION_PATTERN = re.compile(r"^\d+\.\d+\.\d+$") +STABLE_RELEASE_TAG_PATTERN = re.compile(r"^rust-v(\d+\.\d+\.\d+)$") + + +def _version_parts(version: str) -> tuple[int, int, int]: + major, minor, patch = version.split("-", 1)[0].split(".") + return int(major), int(minor), int(patch) @dataclass(frozen=True) @@ -121,6 +131,11 @@ def __init__( self.refresh_interval_seconds = max(float(refresh_interval_seconds), 0.0) self.cache_path = Path(cache_path) if cache_path else Path(get_home_dir()) / MODEL_CACHE_FILE self._session = session or requests.Session() + override = (os.getenv(CLIENT_VERSION_ENV) or "").strip() + if override and not CLIENT_VERSION_PATTERN.fullmatch(override): + raise ValueError(f"{CLIENT_VERSION_ENV} must be a Codex version such as 0.156.0") + self._version_override = override or None + self._client_version = self._version_override or CODEX_MODELS_CLIENT_VERSION self._lock = threading.Lock() self._models: tuple[CatalogModel, ...] = () self._raw_models: list[dict[str, Any]] = [] @@ -171,7 +186,7 @@ def refresh_if_due(self, *, wait_for_refresh: bool = False) -> None: refresh_active = True if wait_for_refresh and refresh_active: - event.wait(FETCH_TIMEOUT_SECONDS + 1) + event.wait(RELEASE_FETCH_TIMEOUT_SECONDS + FETCH_TIMEOUT_SECONDS + 1) def _is_due_locked(self) -> bool: if not self._models or self._fetched_at is None: @@ -201,12 +216,13 @@ def _fetch_and_apply(self) -> None: if not access_token or not account_id: return - response = self._request_models(access_token, account_id) + client_version = self._resolve_client_version() + response = self._request_models(access_token, account_id, client_version) if response.status_code == 401: access_token, account_id = get_effective_chatgpt_auth(force_refresh=True) if not access_token or not account_id: return - response = self._request_models(access_token, account_id) + response = self._request_models(access_token, account_id, client_version) response.raise_for_status() payload = response.json() @@ -223,12 +239,36 @@ def _fetch_and_apply(self) -> None: self._fetched_at = fetched_at self._etag = etag self._account_id = account_id + self._client_version = client_version self._persist_cache() - def _request_models(self, access_token: str, account_id: str) -> requests.Response: + def _resolve_client_version(self) -> str: + if self._version_override: + return self._version_override + try: + response = self._session.get( + CODEX_RELEASE_URL, + headers={"Accept": "application/vnd.github+json"}, + timeout=RELEASE_FETCH_TIMEOUT_SECONDS, + ) + response.raise_for_status() + release = response.json() + if not isinstance(release, dict) or release.get("draft") or release.get("prerelease"): + return self._client_version + tag = release.get("tag_name") + match = STABLE_RELEASE_TAG_PATTERN.fullmatch(tag) if isinstance(tag, str) else None + if match and _version_parts(match.group(1)) > _version_parts(self._client_version): + return match.group(1) + except (requests.RequestException, ValueError, TypeError): + pass + return self._client_version + + def _request_models( + self, access_token: str, account_id: str, client_version: str + ) -> requests.Response: return self._session.get( f"{CHATGPT_CODEX_BASE_URL}/models", - params={"client_version": CODEX_MODELS_CLIENT_VERSION}, + params={"client_version": client_version}, headers={ "Authorization": f"Bearer {access_token}", "Accept": "application/json", @@ -258,6 +298,15 @@ def _load_cache(self) -> None: ): return + cached_version = payload.get("client_version") + if ( + not self._version_override + and isinstance(cached_version, str) + and STABLE_RELEASE_TAG_PATTERN.fullmatch(f"rust-v{cached_version}") + and _version_parts(cached_version) > _version_parts(self._client_version) + ): + self._client_version = cached_version + raw_models = payload.get("models") parsed_models = _parse_models(raw_models) if not parsed_models or not any(model.visibility == "list" for model in parsed_models): @@ -265,7 +314,11 @@ def _load_cache(self) -> None: with self._lock: self._models = parsed_models self._raw_models = [dict(item) for item in raw_models if isinstance(item, dict)] - self._fetched_at = _parse_timestamp(payload.get("fetched_at")) + self._fetched_at = ( + _parse_timestamp(payload.get("fetched_at")) + if cached_version == self._client_version + else None + ) self._etag = payload.get("etag") if isinstance(payload.get("etag"), str) else None self._account_id = cached_account_id @@ -276,7 +329,7 @@ def _persist_cache(self) -> None: if self._fetched_at else None, "etag": self._etag, - "client_version": CODEX_MODELS_CLIENT_VERSION, + "client_version": self._client_version, "account_id": self._account_id, "models": self._raw_models, } diff --git a/chatmock/model_registry.py b/chatmock/model_registry.py index 0768b5ee..45b58341 100644 --- a/chatmock/model_registry.py +++ b/chatmock/model_registry.py @@ -246,21 +246,12 @@ def extract_reasoning_from_model_name(model: str | None) -> dict[str, str] | Non def list_public_models(expose_reasoning_models: bool = False) -> list[str]: catalog = current_model_catalog() - if catalog is not None: - remote_models = catalog.visible_models(wait_for_refresh=True) - if remote_models: - model_ids: list[str] = [] - for model in remote_models: - model_ids.append(model.slug) - if expose_reasoning_models: - model_ids.extend(f"{model.slug}-{effort}" for effort in model.reasoning_efforts) - return model_ids - + remote_models = catalog.visible_models(wait_for_refresh=True) if catalog is not None else () model_ids: list[str] = [] - for spec in _MODEL_SPECS: - model_ids.append(spec.public_id) + for model in remote_models: + model_ids.append(model.slug) if expose_reasoning_models: - model_ids.extend(f"{spec.public_id}-{effort}" for effort in spec.variant_efforts) + model_ids.extend(f"{model.slug}-{effort}" for effort in model.reasoning_efforts) return model_ids diff --git a/chatmock/routes_ollama.py b/chatmock/routes_ollama.py index 70981eea..fe011825 100644 --- a/chatmock/routes_ollama.py +++ b/chatmock/routes_ollama.py @@ -86,6 +86,8 @@ def ollama_tags() -> Response: print("IN GET /api/tags") expose_variants = bool(current_app.config.get("EXPOSE_REASONING_MODELS")) model_ids = list_public_models(expose_reasoning_models=expose_variants) + if not model_ids: + return make_response(jsonify({"error": "Model catalog unavailable"}), 503) models = [] for model_id in model_ids: models.append( diff --git a/chatmock/routes_openai.py b/chatmock/routes_openai.py index 673e22fb..7793dedc 100644 --- a/chatmock/routes_openai.py +++ b/chatmock/routes_openai.py @@ -721,6 +721,8 @@ def responses_create() -> Response: def list_models() -> Response: expose_variants = bool(current_app.config.get("EXPOSE_REASONING_MODELS")) model_ids = list_public_models(expose_reasoning_models=expose_variants) + if not model_ids: + return make_response(jsonify({"error": {"message": "Model catalog unavailable"}}), 503) data = [{"id": mid, "object": "model", "owned_by": "owner"} for mid in model_ids] models = {"object": "list", "data": data} resp = make_response(jsonify(models), 200) diff --git a/tests/test_model_catalog.py b/tests/test_model_catalog.py new file mode 100644 index 00000000..224fa3c7 --- /dev/null +++ b/tests/test_model_catalog.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +import requests + +from chatmock.model_catalog import ( + CLIENT_VERSION_ENV, + CODEX_MODELS_CLIENT_VERSION, + CODEX_RELEASE_URL, + CatalogModel, + ModelCatalog, +) +from chatmock.app import create_app +from chatmock.fast_mode import supports_priority_service_tier + + +def response(payload: dict, status: int = 200) -> Mock: + result = Mock() + result.status_code = status + result.json.return_value = payload + result.headers = {} + result.raise_for_status.side_effect = ( + requests.HTTPError("release unavailable") if status >= 400 else None + ) + return result + + +class ModelCatalogTests(unittest.TestCase): + def test_latest_stable_release_supplies_catalog_client_version(self) -> None: + session = Mock() + session.get.side_effect = [ + response({"tag_name": "rust-v0.157.0", "prerelease": False, "draft": False}), + response({"models": [{"slug": "future-model", "visibility": "list"}]}), + ] + with tempfile.TemporaryDirectory() as directory: + catalog = ModelCatalog( + enabled=False, session=session, cache_path=Path(directory) / "models.json" + ) + with patch("chatmock.model_catalog.get_effective_chatgpt_auth", return_value=("token", "account")): + catalog._fetch_and_apply() + + self.assertEqual(session.get.call_args_list[0].args[0], CODEX_RELEASE_URL) + self.assertEqual(session.get.call_args_list[1].kwargs["params"], {"client_version": "0.157.0"}) + self.assertEqual([model.slug for model in catalog.visible_models()], ["future-model"]) + + def test_invalid_release_or_github_failure_uses_verified_fallback(self) -> None: + for release in ( + response({"tag_name": "rust-v0.157.0-alpha.9", "prerelease": True}), + response({"tag_name": "unrelated-v2.0.0", "prerelease": False}), + response({}, 503), + ): + with self.subTest(release=release): + session = Mock() + session.get.return_value = release + catalog = ModelCatalog(enabled=False, session=session) + self.assertEqual(catalog._resolve_client_version(), CODEX_MODELS_CLIENT_VERSION) + + def test_environment_override_skips_github(self) -> None: + session = Mock() + with patch.dict("os.environ", {CLIENT_VERSION_ENV: "0.155.0"}): + catalog = ModelCatalog(enabled=False, session=session) + self.assertEqual(catalog._resolve_client_version(), "0.155.0") + session.get.assert_not_called() + + def test_invalid_environment_override_fails_loudly(self) -> None: + for version in ("latest", "0.157.0-alpha.9"): + with self.subTest(version=version): + with patch.dict("os.environ", {CLIENT_VERSION_ENV: version}): + with self.assertRaises(ValueError): + ModelCatalog(enabled=False) + + def test_cache_from_older_version_is_refreshed_without_losing_last_models(self) -> None: + with tempfile.TemporaryDirectory() as directory: + cache_path = Path(directory) / "models.json" + cache_path.write_text( + json.dumps({ + "account_id": "account", + "client_version": "0.146.0", + "fetched_at": "2026-09-23T00:00:00Z", + "models": [{"slug": "old-model", "visibility": "list"}], + }), + encoding="utf-8", + ) + with patch("chatmock.model_catalog._account_id_from_auth_file", return_value="account"): + catalog = ModelCatalog(cache_path=cache_path) + self.assertEqual([model.slug for model in catalog._models], ["old-model"]) + self.assertIsNone(catalog._fetched_at) + self.assertEqual(catalog._client_version, CODEX_MODELS_CLIENT_VERSION) + + def test_future_model_priority_uses_catalog_metadata(self) -> None: + app = create_app(model_sync=False) + catalog = app.extensions["chatmock_model_catalog"] + catalog._models = ( + CatalogModel("future-model", ("low",), frozenset(("priority",)), 1, "list", True), + CatalogModel("ordinary-model", ("low",), frozenset(), 2, "list", True), + ) + with app.app_context(): + self.assertTrue(supports_priority_service_tier("future-model")) + self.assertFalse(supports_priority_service_tier("ordinary-model")) + self.assertIsNone(supports_priority_service_tier("unlisted-model")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_routes.py b/tests/test_routes.py index a4906705..aebb6952 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -8,6 +8,7 @@ from unittest.mock import patch from chatmock.app import create_app +from chatmock.model_catalog import CatalogModel from chatmock.session import reset_session_state from websockets.sync.client import connect as ws_connect @@ -54,28 +55,25 @@ def setUp(self) -> None: self.app = create_app(model_sync=False) self.client = self.app.test_client() - def test_openai_models_list(self) -> None: + def test_model_lists_require_a_discovered_catalog(self) -> None: response = self.client.get("/v1/models") - body = response.get_json() - self.assertEqual(response.status_code, 200) - model_ids = [item["id"] for item in body["data"]] - self.assertIn("gpt-5.4", model_ids) - self.assertIn("gpt-5.4-mini", model_ids) - self.assertIn("gpt-5.3-codex-spark", model_ids) - self.assertIn("gpt-5.6-sol", model_ids) - self.assertIn("gpt-5.6-terra", model_ids) - self.assertIn("gpt-5.6-luna", model_ids) - - def test_ollama_tags_list(self) -> None: - response = self.client.get("/api/tags") - body = response.get_json() - self.assertEqual(response.status_code, 200) - model_names = [item["name"] for item in body["models"]] - self.assertIn("gpt-5.4", model_names) - self.assertIn("gpt-5.4-mini", model_names) - self.assertIn("gpt-5.6-sol", model_names) - self.assertIn("gpt-5.6-terra", model_names) - self.assertIn("gpt-5.6-luna", model_names) + self.assertEqual(response.status_code, 503) + self.assertEqual(self.client.get("/api/tags").status_code, 503) + + @patch("chatmock.model_catalog.ModelCatalog.visible_models") + def test_model_lists_use_discovered_models_without_registration(self, mock_visible) -> None: + mock_visible.return_value = ( + CatalogModel("future-model", ("low", "high"), frozenset(("priority",)), 1, "list", True), + ) + app = create_app(model_sync=True, expose_reasoning_models=True) + client = app.test_client() + openai = client.get("/v1/models") + ollama = client.get("/api/tags") + self.assertEqual(openai.status_code, 200) + self.assertEqual(ollama.status_code, 200) + expected = ["future-model", "future-model-low", "future-model-high"] + self.assertEqual([item["id"] for item in openai.get_json()["data"]], expected) + self.assertEqual([item["name"] for item in ollama.get_json()["models"]], expected) @patch("chatmock.routes_openai.start_upstream_request") def test_chat_completions(self, mock_start) -> None: @@ -159,8 +157,9 @@ def test_ollama_chat_honors_debug_model_override(self, mock_start) -> None: self.assertEqual(mock_start.call_args.args[0], "gpt-5.4") self.assertEqual(body["model"], "gpt-5.4") + @patch("chatmock.fast_mode.model_supports_service_tier", return_value=True) @patch("chatmock.routes_openai.start_upstream_request") - def test_chat_completions_fast_mode_sets_priority_service_tier(self, mock_start) -> None: + def test_chat_completions_fast_mode_sets_priority_service_tier(self, mock_start, _mock_support) -> None: mock_start.return_value = ( FakeUpstream( [ @@ -205,8 +204,9 @@ def test_chat_completions_fast_mode_false_overrides_server_default(self, mock_st self.assertEqual(response.status_code, 200) self.assertIsNone(mock_start.call_args.kwargs["service_tier"]) + @patch("chatmock.fast_mode.model_supports_service_tier", return_value=False) @patch("chatmock.routes_openai.start_upstream_request") - def test_chat_completions_rejects_unsupported_explicit_fast_mode(self, mock_start) -> None: + def test_chat_completions_rejects_unsupported_explicit_fast_mode(self, mock_start, _mock_support) -> None: response = self.client.post( "/v1/chat/completions", json={ @@ -551,8 +551,9 @@ def test_responses_route_stream_passthrough(self, mock_start) -> None: self.assertEqual(response.status_code, 200) self.assertIn("response.output_text.delta", response.get_data(as_text=True)) + @patch("chatmock.fast_mode.model_supports_service_tier", return_value=False) @patch("chatmock.routes_openai.start_upstream_raw_request") - def test_responses_route_rejects_unsupported_explicit_priority(self, mock_start) -> None: + def test_responses_route_rejects_unsupported_explicit_priority(self, mock_start, _mock_support) -> None: response = self.client.post( "/v1/responses", json={"model": "gpt-5.3-codex", "input": "hello", "service_tier": "priority"}, @@ -562,9 +563,10 @@ def test_responses_route_rejects_unsupported_explicit_priority(self, mock_start) self.assertIn("Fast mode is not supported", body["error"]["message"]) mock_start.assert_not_called() + @patch("chatmock.fast_mode.model_supports_service_tier", return_value=True) @patch("chatmock.websocket_routes.get_effective_chatgpt_auth", return_value=("token", "acct")) @patch("chatmock.websocket_routes.connect_upstream_websocket") - def test_responses_websocket_rewrites_response_create(self, mock_connect, _mock_auth) -> None: + def test_responses_websocket_rewrites_response_create(self, mock_connect, _mock_auth, _mock_support) -> None: class FakeUpstreamWebsocket: def __init__(self) -> None: self.sent: list[str] = []