""" Arena Dashboard — FastAPI backend Таблица моделей OmniRoute с Arena Score (text + agent) и liveness-статусом. """ from __future__ import annotations import json import os import sqlite3 import time import urllib.request from contextlib import asynccontextmanager from datetime import datetime, timezone from typing import Optional from fastapi import FastAPI, Query from fastapi.responses import HTMLResponse, JSONResponse from fastapi.staticfiles import StaticFiles DB_PATH = os.environ.get("DB_PATH", "/data/arena.db") OMNIROUTE_URL = os.environ.get("OMNIROUTE_URL", "http://192.168.1.101:20128") OMNIROUTE_KEY = os.environ.get("OMNIROUTE_API_KEY", "") ARENA_CACHE_TEXT = os.environ.get("ARENA_CACHE_TEXT", "/data/arena_text.json") ARENA_CACHE_AGENT = os.environ.get("ARENA_CACHE_AGENT", "/data/arena_agent.json") # ─── DB setup ─────────────────────────────────────────────────────────────── def get_db() -> sqlite3.Connection: db = sqlite3.connect(DB_PATH, check_same_thread=False) db.row_factory = sqlite3.Row return db def init_db(): db = get_db() # Автомиграция таблицы models если создана со старой схемой try: cols = [r[1] for r in db.execute("PRAGMA table_info(models)").fetchall()] if cols and "raw_base" not in cols: db.execute("DROP TABLE models") except Exception: pass db.executescript(""" CREATE TABLE IF NOT EXISTS arena_text_leaderboard ( rank INTEGER, model_name TEXT PRIMARY KEY, norm_name TEXT NOT NULL, clean_name TEXT NOT NULL, provider TEXT, score TEXT, preliminary INTEGER, rank_spread TEXT, votes INTEGER, price TEXT, context TEXT, fetched_at TEXT ); CREATE TABLE IF NOT EXISTS arena_agent_leaderboard ( rank INTEGER, model_name TEXT PRIMARY KEY, norm_name TEXT NOT NULL, clean_name TEXT NOT NULL, provider TEXT, net_improvement TEXT, confirmed_success TEXT, praise_vs_complaint TEXT, steerability TEXT, bash_recovery TEXT, tool_hallucination TEXT, sessions INTEGER, cost_per_task TEXT, output_tokens_per_task TEXT, price TEXT, fetched_at TEXT ); CREATE TABLE IF NOT EXISTS models ( id TEXT PRIMARY KEY, provider TEXT NOT NULL, name TEXT NOT NULL, raw_base TEXT NOT NULL, norm_base TEXT NOT NULL, clean_base TEXT NOT NULL, text_model_match TEXT, agent_model_match TEXT, updated_at TEXT, FOREIGN KEY(text_model_match) REFERENCES arena_text_leaderboard(model_name), FOREIGN KEY(agent_model_match) REFERENCES arena_agent_leaderboard(model_name) ); CREATE TABLE IF NOT EXISTS checks ( id INTEGER PRIMARY KEY AUTOINCREMENT, model_id TEXT NOT NULL, checked_at TEXT NOT NULL, ok INTEGER NOT NULL, latency_ms INTEGER ); CREATE INDEX IF NOT EXISTS idx_checks_model ON checks(model_id); CREATE INDEX IF NOT EXISTS idx_checks_time ON checks(checked_at); CREATE INDEX IF NOT EXISTS idx_at_norm ON arena_text_leaderboard(norm_name); CREATE INDEX IF NOT EXISTS idx_ag_norm ON arena_agent_leaderboard(norm_name); """) db.commit() db.close() # ─── Arena data ingestion into DB ─────────────────────────────────────────── def ingest_arena_tables(db: sqlite3.Connection): """Импортирует сырые JSON кеши арены прямо в реляционные таблицы SQLite.""" now = datetime.now(timezone.utc).isoformat() if os.path.exists(ARENA_CACHE_TEXT): try: with open(ARENA_CACHE_TEXT) as f: tdata = json.load(f) for m in tdata.get("models", []): raw_m = m["model"] m_only = raw_m.split(" · ")[0] m_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", m_only, flags=re.I) clean = re.sub(r"\s*\([^)]+\)", "", m_only) db.execute(""" INSERT INTO arena_text_leaderboard ( rank, model_name, norm_name, clean_name, provider, score, preliminary, rank_spread, votes, price, context, fetched_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(model_name) DO UPDATE SET rank = excluded.rank, norm_name = excluded.norm_name, clean_name = excluded.clean_name, provider = excluded.provider, score = excluded.score, preliminary = excluded.preliminary, rank_spread = excluded.rank_spread, votes = excluded.votes, price = excluded.price, context = excluded.context, fetched_at = excluded.fetched_at """, ( m.get("rank"), raw_m, _norm(m_only), _norm(clean), m.get("provider"), m.get("score"), int(bool(m.get("preliminary"))), m.get("rank_spread"), m.get("votes"), m.get("price"), m.get("context"), now )) except Exception as e: print(f"[ingest] text leaderboard error: {e}") if os.path.exists(ARENA_CACHE_AGENT): try: with open(ARENA_CACHE_AGENT) as f: adata = json.load(f) for m in adata.get("models", []): raw_m = m["model"] m_only = raw_m.split(" · ")[0] m_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", m_only, flags=re.I) clean = re.sub(r"\s*\([^)]+\)", "", m_only) db.execute(""" INSERT INTO arena_agent_leaderboard ( rank, model_name, norm_name, clean_name, provider, net_improvement, confirmed_success, praise_vs_complaint, steerability, bash_recovery, tool_hallucination, sessions, cost_per_task, output_tokens_per_task, price, fetched_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(model_name) DO UPDATE SET rank = excluded.rank, norm_name = excluded.norm_name, clean_name = excluded.clean_name, provider = excluded.provider, net_improvement = excluded.net_improvement, confirmed_success = excluded.confirmed_success, praise_vs_complaint = excluded.praise_vs_complaint, steerability = excluded.steerability, bash_recovery = excluded.bash_recovery, tool_hallucination = excluded.tool_hallucination, sessions = excluded.sessions, cost_per_task = excluded.cost_per_task, output_tokens_per_task = excluded.output_tokens_per_task, price = excluded.price, fetched_at = excluded.fetched_at """, ( m.get("rank"), raw_m, _norm(m_only), _norm(clean), m.get("provider"), m.get("net_improvement"), m.get("confirmed_success"), m.get("praise_vs_complaint"), m.get("steerability"), m.get("bash_recovery"), m.get("tool_hallucination"), m.get("sessions"), m.get("cost_per_task"), m.get("output_tokens_per_task"), m.get("price"), now )) except Exception as e: print(f"[ingest] agent leaderboard error: {e}") db.commit() # ─── OmniRoute helpers ──────────────────────────────────────────────────────── def fetch_omni_models() -> list[dict]: req = urllib.request.Request( f"{OMNIROUTE_URL}/v1/models", headers={"Authorization": f"Bearer {OMNIROUTE_KEY}"}, ) with urllib.request.urlopen(req, timeout=15) as r: data = json.loads(r.read()) return data.get("data", []) def is_skip(model_id: str) -> bool: return any(x in model_id.lower() for x in [ "image", "embed", "flux", "tts", "stt", "audio", "whisper", "vision", "guard", "safety", "moderat", ]) # ─── Arena matching (simple) ───────────────────────────────────────────────── import re def _norm(s: str) -> str: return re.sub(r"[^a-z0-9]", "", s.lower()) def _norm_keep(s: str) -> str: return re.sub(r"[^a-z0-9\-]", "", s.lower()) def arena_match(model_id: str, arena: dict[str, dict]) -> Optional[dict]: """Сопоставить модель OmniRoute с записью арены.""" raw = model_id.split("/")[-1].lower() base = re.sub(r"-(it|preview)(:[^:]*)?$", "", raw) base = re.sub(r"(:[^:]*)$", "", base) # 1) точное совпадение нормализованного имени bn = _norm(base) for k, v in arena.items(): # убираем вендор-хвост " Anthropic · Proprietary" из названия модели арены model_name_only = v.get("model", k).split(" · ")[0] model_name_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", model_name_only, flags=re.I) an = _norm(model_name_only) if an == bn: return v # 2) без суффиксов тиров cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", base) cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", cleaned) cn = _norm(cleaned) for k, v in arena.items(): model_name_only = v.get("model", k).split(" · ")[0] model_name_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", model_name_only, flags=re.I) # убираем скобки тиров типа (Max), (High) m_cleaned = re.sub(r"\s*\([^)]+\)", "", model_name_only) if _norm(m_cleaned) == cn: return v # 3) containment for candidate in (base, cleaned): gk = _norm_keep(candidate) if len(gk) >= 10 and (re.search(r"\d", gk) or "-" in gk): for k, v in arena.items(): model_name_only = v.get("model", k).split(" · ")[0] am = _norm_keep(model_name_only) if gk == am or am.endswith("-" + gk) or ("-" + gk) in am or (gk in am and len(gk) >= 12): return v return None # ─── model sync ────────────────────────────────────────────────────────────── def sync_models(): """Обновить список моделей в DB из OmniRoute + найти связи с таблицами арены.""" db = get_db() ingest_arena_tables(db) try: omni = fetch_omni_models() except Exception as e: print(f"[sync] OmniRoute error: {e}") db.close() return # Загружаем нормализованные имена из таблиц арены для сопоставления text_rows = db.execute("SELECT model_name, norm_name, clean_name FROM arena_text_leaderboard").fetchall() agent_rows = db.execute("SELECT model_name, norm_name, clean_name FROM arena_agent_leaderboard").fetchall() def find_match(base: str, cleaned: str, rows) -> Optional[str]: bn = _norm(base) for r in rows: if r["norm_name"] == bn or r["clean_name"] == bn: return r["model_name"] cn = _norm(cleaned) for r in rows: if r["clean_name"] == cn: return r["model_name"] for candidate in (base, cleaned): gk = _norm_keep(candidate) if len(gk) >= 10 and (re.search(r"\d", gk) or "-" in gk): for r in rows: am = _norm_keep(r["model_name"]) if gk == am or am.endswith("-" + gk) or ("-" + gk) in am or (gk in am and len(gk) >= 12): return r["model_name"] return None now = datetime.now(timezone.utc).isoformat() for m in omni: mid = m["id"] if is_skip(mid): continue # openrouter — только :free if mid.startswith("openrouter/") and not mid.endswith(":free"): continue provider = mid.split("/")[0] raw = mid.split("/")[-1].lower() base = re.sub(r"-(it|preview)(:[^:]*)?$", "", raw) base = re.sub(r"(:[^:]*)$", "", base) cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", base) cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", cleaned) text_match = find_match(base, cleaned, text_rows) agent_match = find_match(base, cleaned, agent_rows) db.execute(""" INSERT INTO models (id, provider, name, raw_base, norm_base, clean_base, text_model_match, agent_model_match, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET text_model_match = excluded.text_model_match, agent_model_match = excluded.agent_model_match, updated_at = excluded.updated_at """, ( mid, provider, mid, base, _norm(base), _norm(cleaned), text_match, agent_match, now )) db.commit() db.close() print(f"[sync] synced {len(omni)} models at {now} with SQL foreign keys") # ─── Stats helpers ──────────────────────────────────────────────────────────── def model_stats(db: sqlite3.Connection, model_id: str, hours: int = 24) -> dict: """Последний статус и % успехов за N часов.""" cutoff = datetime.fromtimestamp(time.time() - hours * 3600, timezone.utc).isoformat() last = db.execute( "SELECT ok, latency_ms, checked_at FROM checks WHERE model_id=? ORDER BY checked_at DESC LIMIT 1", (model_id,) ).fetchone() rows = db.execute( "SELECT ok FROM checks WHERE model_id=? AND checked_at > ?", (model_id, cutoff) ).fetchall() total = len(rows) success = sum(1 for r in rows if r["ok"]) pct = round(100 * success / total) if total else None return { "last_ok": bool(last["ok"]) if last else None, "last_latency_ms": last["latency_ms"] if last else None, "last_checked": last["checked_at"] if last else None, "uptime_pct": pct, "checks_count": total, } # ─── Lifespan ──────────────────────────────────────────────────────────────── @asynccontextmanager async def lifespan(app: FastAPI): init_db() sync_models() yield app = FastAPI(title="Arena Dashboard", lifespan=lifespan) # ─── API routes ─────────────────────────────────────────────────────────────── @app.get("/api/models") def api_models( provider: Optional[str] = Query(None), arena_only: bool = Query(False), sort: str = Query("arena_text_rank"), ): db = get_db() # SQL JOIN таблиц арены с моделями OmniRoute! q = """ SELECT m.id, m.provider, m.name, t.score as arena_text_score, t.rank as arena_text_rank, a.net_improvement as arena_agent_score, a.rank as arena_agent_rank, t.votes as arena_text_votes, t.price as arena_text_price FROM models m LEFT JOIN arena_text_leaderboard t ON m.text_model_match = t.model_name LEFT JOIN arena_agent_leaderboard a ON m.agent_model_match = a.model_name WHERE 1=1 """ params: list = [] if provider: q += " AND m.provider = ?" params.append(provider) if arena_only: q += " AND t.score IS NOT NULL" rows = db.execute(q, params).fetchall() result = [] for r in rows: mid = r["id"] stats = model_stats(db, mid) result.append({ "id": mid, "provider": r["provider"], "name": r["name"], "arena_text_score": r["arena_text_score"], "arena_text_rank": r["arena_text_rank"], "arena_agent_score": r["arena_agent_score"], "arena_agent_rank": r["arena_agent_rank"], "arena_text_votes": r["arena_text_votes"], "arena_text_price": r["arena_text_price"], **stats, }) db.close() # сортировка def sort_key(x): if sort == "arena_text_rank": return (x["arena_text_rank"] is None, x["arena_text_rank"] or 9999) if sort == "uptime_pct": return (x["uptime_pct"] is None, -(x["uptime_pct"] or 0)) if sort == "provider": return x["provider"] return x.get(sort) or "" result.sort(key=sort_key) return JSONResponse(result) @app.get("/api/providers") def api_providers(): db = get_db() rows = db.execute("SELECT DISTINCT provider FROM models ORDER BY provider").fetchall() db.close() return [r["provider"] for r in rows] @app.post("/api/sync") def api_sync(): sync_models() return {"ok": True} @app.get("/api/check/{model_id:path}") def api_check(model_id: str): """Мгновенная проверка одной модели.""" import time as _t t0 = _t.time() ok = False latency = None try: body = json.dumps({ "model": model_id, "messages": [{"role": "user", "content": "What is 2+2? Reply with just the number."}], "max_tokens": 5, }).encode() req = urllib.request.Request( f"{OMNIROUTE_URL}/v1/chat/completions", data=body, headers={ "Authorization": f"Bearer {OMNIROUTE_KEY}", "Content-Type": "application/json", }, ) with urllib.request.urlopen(req, timeout=30) as r: resp = json.loads(r.read()) content = resp["choices"][0]["message"]["content"] ok = bool(content.strip()) latency = int((_t.time() - t0) * 1000) except Exception as e: print(f"[check] {model_id}: {e}") now = datetime.now(timezone.utc).isoformat() db = get_db() db.execute( "INSERT INTO checks (model_id, checked_at, ok, latency_ms) VALUES (?,?,?,?)", (model_id, now, int(ok), latency), ) db.commit() db.close() return {"model_id": model_id, "ok": ok, "latency_ms": latency, "checked_at": now} @app.get("/", response_class=HTMLResponse) def index(): with open("/app/app/static/index.html") as f: return f.read() app.mount("/static", StaticFiles(directory="/app/app/static"), name="static")