From 631b44666f20f0dcbf3d51dbaa017de7780569a6 Mon Sep 17 00:00:00 2001 From: ai_coder Date: Tue, 27 Oct 2026 09:44:23 +0000 Subject: [PATCH] fix: hide stale and effort-preset models without losing checks --- app/main.py | 51 +++++++++++++++++++++++++++++++++---------- app/probe.py | 6 ++++- tests/conftest.py | 3 +++ tests/test_catalog.py | 46 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 13 deletions(-) create mode 100644 tests/test_catalog.py diff --git a/app/main.py b/app/main.py index 7f1095f..7ed40ed 100644 --- a/app/main.py +++ b/app/main.py @@ -6,6 +6,7 @@ from __future__ import annotations import json import os +import re import sqlite3 import time import urllib.request @@ -95,6 +96,7 @@ def init_db(): text_model_match TEXT, agent_model_match TEXT, updated_at TEXT, + active INTEGER NOT NULL DEFAULT 1, FOREIGN KEY(text_model_match) REFERENCES arena_text_leaderboard(model_name), FOREIGN KEY(agent_model_match) REFERENCES arena_agent_leaderboard(model_name) ); @@ -133,6 +135,11 @@ def init_db(): if col not in cols: db.execute(f"ALTER TABLE checks ADD COLUMN {col} {decl}") + # Историю checks не трогаем; видимость модели живёт отдельно. + model_cols = [r[1] for r in db.execute("PRAGMA table_info(models)").fetchall()] + if "active" not in model_cols: + db.execute("ALTER TABLE models ADD COLUMN active INTEGER NOT NULL DEFAULT 1") + db.commit() db.close() @@ -232,7 +239,9 @@ def fetch_omni_models() -> list[dict]: ) with urllib.request.urlopen(req, timeout=15) as r: data = json.loads(r.read()) - return data.get("data", []) + if not isinstance(data, dict) or not isinstance(data.get("data"), list): + raise ValueError("OmniRoute returned an invalid models catalog") + return data["data"] def is_skip(model_id: str) -> bool: @@ -242,11 +251,17 @@ def is_skip(model_id: str) -> bool: ]) +def is_effort_variant(model_id: str, catalog_ids: set[str]) -> bool: + """Пресет только при наличии базового ID на том же маршруте. + + Так не скрываем самостоятельные модели вроде qwen3.8-max. + """ + base = re.sub(r"-(?:low|medium|high|xhigh|max|ultra|none)(?=(:free)?$)", "", model_id) + return base != model_id and base in catalog_ids + + + # ─── Arena matching (simple) ───────────────────────────────────────────────── - -import re - - def _norm(s: str) -> str: return re.sub(r"[^a-z0-9]", "", s.lower()) @@ -332,7 +347,12 @@ def sync_models(): return None now = datetime.now(timezone.utc).isoformat() - + if not omni: + print("[sync] OmniRoute returned an empty catalog; keeping previous visibility") + db.close() + return + catalog_ids = {m["id"] for m in omni} + seen = set() for m in omni: mid = m["id"] if is_skip(mid): @@ -340,6 +360,9 @@ def sync_models(): # openrouter — только :free if mid.startswith("openrouter/") and not mid.endswith(":free"): continue + if is_effort_variant(mid, catalog_ids): + continue + seen.add(mid) provider = mid.split("/")[0] raw = mid.split("/")[-1].lower() @@ -353,20 +376,24 @@ def sync_models(): db.execute(""" INSERT INTO models (id, provider, name, raw_base, norm_base, clean_base, - text_model_match, agent_model_match, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + text_model_match, agent_model_match, updated_at, active) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1) ON CONFLICT(id) DO UPDATE SET text_model_match = excluded.text_model_match, agent_model_match = excluded.agent_model_match, - updated_at = excluded.updated_at + updated_at = excluded.updated_at, + active = 1 """, ( mid, provider, mid, base, _norm(base), _norm(cleaned), text_match, agent_match, now )) + # Снимок каталога целиком: устаревшие/пресетные строки сохраняются для + # истории checks, но не попадают в список и следующий cron-прогон. + db.execute("UPDATE models SET active = 0 WHERE updated_at IS NULL OR updated_at != ?", (now,)) db.commit() db.close() - print(f"[sync] synced {len(omni)} models at {now} with SQL foreign keys") + print(f"[sync] active {len(seen)} models from {len(omni)} OmniRoute entries at {now}") # ─── Stats helpers ──────────────────────────────────────────────────────────── @@ -538,7 +565,7 @@ def api_models( FROM models m LEFT JOIN arena_text_leaderboard t ON m.text_model_match = t.model_name LEFT JOIN arena_agent_leaderboard a ON m.agent_model_match = a.model_name - WHERE 1=1 + WHERE m.active = 1 """ params: list = [] if provider: @@ -585,7 +612,7 @@ def api_models( @app.get("/api/providers") def api_providers(): db = get_db() - rows = db.execute("SELECT DISTINCT provider FROM models ORDER BY provider").fetchall() + rows = db.execute("SELECT DISTINCT provider FROM models WHERE active = 1 ORDER BY provider").fetchall() db.close() return [r["provider"] for r in rows] diff --git a/app/probe.py b/app/probe.py index 4762360..6cbc86e 100644 --- a/app/probe.py +++ b/app/probe.py @@ -81,7 +81,11 @@ def _ensure_usage_columns(db: sqlite3.Connection): def run(): db = sqlite3.connect(DB_PATH, check_same_thread=False) _ensure_usage_columns(db) - models = [r[0] for r in db.execute("SELECT id FROM models").fetchall()] + model_cols = [r[1] for r in db.execute("PRAGMA table_info(models)")] + if "active" not in model_cols: + db.execute("ALTER TABLE models ADD COLUMN active INTEGER NOT NULL DEFAULT 1") + db.commit() + models = [r[0] for r in db.execute("SELECT id FROM models WHERE active = 1").fetchall()] db.close() print(f"[probe] checking {len(models)} models @ {datetime.now(timezone.utc).isoformat()}") diff --git a/tests/conftest.py b/tests/conftest.py index 8c77ed3..a767b9b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -125,6 +125,9 @@ def _reset_omni(omni_server): """Изолировать тесты: состояние мока не должно протекать между тестами.""" with omni_server.lock: omni_server.state["per_model"] = {} + omni_server.state["models"] = {"data": [ + {"id": "zai/test-model-a"}, {"id": "agy/test-model-b"}, + ]} omni_server.state["global_delay"] = 0.0 omni_server.state["completions"] = {"status": 200, "usage": None, "content": "4"} omni_server.requests.clear() diff --git a/tests/test_catalog.py b/tests/test_catalog.py new file mode 100644 index 0000000..8106fc2 --- /dev/null +++ b/tests/test_catalog.py @@ -0,0 +1,46 @@ +"""Каталог OmniRoute: пропавшие ID и effort-пресеты не должны загрязнять витрину.""" +import sqlite3 + + +def test_sync_hides_stale_and_effort_but_preserves_history(core, omni_server): + db = sqlite3.connect(core.db_path) + db.execute("INSERT INTO checks (model_id,checked_at,ok) VALUES ('zai/test-model-a','2026-10-26T00:00:00+00:00',1)") + db.commit() + db.close() + omni_server.state["models"] = {"data": [ + {"id": "agy/test-model-b"}, + {"id": "agy/test-model-b-high"}, + {"id": "cx/gpt-5.6-terra"}, + {"id": "cx/gpt-5.6-terra-low"}, + {"id": "openrouter/foo/model-high:free"}, + {"id": "openrouter/foo/model:free"}, + {"id": "agy/kimi-highway"}, + {"id": "freellmapi/qwen3.8-max"}, + ]} + assert core.client.post("/api/sync").status_code == 200 + ids = {m["id"] for m in core.client.get("/api/models").json()} + assert ids == {"agy/test-model-b", "cx/gpt-5.6-terra", "openrouter/foo/model:free", "agy/kimi-highway", "freellmapi/qwen3.8-max"} + assert "zai" not in core.client.get("/api/providers").json() + db = sqlite3.connect(core.db_path) + assert db.execute("SELECT active FROM models WHERE id='zai/test-model-a'").fetchone()[0] == 0 + assert db.execute("SELECT COUNT(*) FROM checks WHERE model_id='zai/test-model-a'").fetchone()[0] == 1 + db.close() + omni_server.requests.clear() + core.probe.run() + requested = {x["body"].get("model") for x in omni_server.requests if x["path"] == "/v1/chat/completions"} + assert requested == ids + # ID возвращается в каталог — снова активен, история на месте. + omni_server.state["models"]["data"].append({"id": "zai/test-model-a"}) + core.client.post("/api/sync") + assert "zai/test-model-a" in {m["id"] for m in core.client.get("/api/models").json()} + db = sqlite3.connect(core.db_path) + assert db.execute("SELECT active FROM models WHERE id='zai/test-model-a'").fetchone()[0] == 1 + assert db.execute("SELECT COUNT(*) FROM checks WHERE model_id='zai/test-model-a'").fetchone()[0] == 1 + db.close() + + +def test_empty_catalog_does_not_hide_everything(core, omni_server): + original = {m["id"] for m in core.client.get("/api/models").json()} + omni_server.state["models"] = {"data": []} + core.client.post("/api/sync") + assert {m["id"] for m in core.client.get("/api/models").json()} == original