From 296fa9c7ad67fb446a4e122b28e42da7ecddb197 Mon Sep 17 00:00:00 2001 From: ai_coder Date: Fri, 16 Oct 2026 15:34:50 +0000 Subject: [PATCH] feat: store arena leaderboards in relational SQLite tables and JOIN with models --- app/main.py | 235 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 189 insertions(+), 46 deletions(-) diff --git a/app/main.py b/app/main.py index e8934f0..04a1587 100644 --- a/app/main.py +++ b/app/main.py @@ -35,15 +35,52 @@ def get_db() -> sqlite3.Connection: def init_db(): db = get_db() db.executescript(""" + CREATE TABLE IF NOT EXISTS arena_text_leaderboard ( + rank INTEGER, + model_name TEXT PRIMARY KEY, + norm_name TEXT NOT NULL, + clean_name TEXT NOT NULL, + provider TEXT, + score TEXT, + preliminary INTEGER, + rank_spread TEXT, + votes INTEGER, + price TEXT, + context TEXT, + fetched_at TEXT + ); + + CREATE TABLE IF NOT EXISTS arena_agent_leaderboard ( + rank INTEGER, + model_name TEXT PRIMARY KEY, + norm_name TEXT NOT NULL, + clean_name TEXT NOT NULL, + provider TEXT, + net_improvement TEXT, + confirmed_success TEXT, + praise_vs_complaint TEXT, + steerability TEXT, + bash_recovery TEXT, + tool_hallucination TEXT, + sessions INTEGER, + cost_per_task TEXT, + output_tokens_per_task TEXT, + price TEXT, + fetched_at TEXT + ); + CREATE TABLE IF NOT EXISTS models ( id TEXT PRIMARY KEY, provider TEXT NOT NULL, name TEXT NOT NULL, - arena_text_score TEXT, - arena_text_rank INTEGER, - arena_agent_score TEXT, - arena_agent_rank INTEGER, - updated_at TEXT + raw_base TEXT NOT NULL, + norm_base TEXT NOT NULL, + clean_base TEXT NOT NULL, + text_model_match TEXT, + agent_model_match TEXT, + updated_at TEXT, + FOREIGN KEY(text_model_match) REFERENCES arena_text_leaderboard(model_name), + FOREIGN KEY(agent_model_match) REFERENCES arena_agent_leaderboard(model_name) ); CREATE TABLE IF NOT EXISTS checks ( @@ -56,26 +93,97 @@ def init_db(): CREATE INDEX IF NOT EXISTS idx_checks_model ON checks(model_id); CREATE INDEX IF NOT EXISTS idx_checks_time ON checks(checked_at); + CREATE INDEX IF NOT EXISTS idx_at_norm ON arena_text_leaderboard(norm_name); + CREATE INDEX IF NOT EXISTS idx_ag_norm ON arena_agent_leaderboard(norm_name); """) db.commit() db.close() -# ─── Arena data loading ────────────────────────────────────────────────────── +# ─── Arena data ingestion into DB ─────────────────────────────────────────── -def load_arena_cache(path: str) -> dict[str, dict]: - """Возвращает dict {normalized_model_name: {rank, score}} из кеш-файла арены.""" - if not os.path.exists(path): - return {} - try: - with open(path) as f: - data = json.load(f) - result = {} - for m in data.get("models", []): - result[m["model"].lower()] = m - return result - except Exception: - return {} +def ingest_arena_tables(db: sqlite3.Connection): + """Импортирует сырые JSON кеши арены прямо в реляционные таблицы SQLite.""" + now = datetime.now(timezone.utc).isoformat() + + if os.path.exists(ARENA_CACHE_TEXT): + try: + with open(ARENA_CACHE_TEXT) as f: + tdata = json.load(f) + for m in tdata.get("models", []): + raw_m = m["model"] + m_only = raw_m.split(" · ")[0] + m_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", m_only, flags=re.I) + clean = re.sub(r"\s*\([^)]+\)", "", m_only) + + db.execute(""" + INSERT INTO arena_text_leaderboard ( + rank, model_name, norm_name, clean_name, provider, score, + preliminary, rank_spread, votes, price, context, fetched_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(model_name) DO UPDATE SET + rank = excluded.rank, + norm_name = excluded.norm_name, + clean_name = excluded.clean_name, + provider = excluded.provider, + score = excluded.score, + preliminary = excluded.preliminary, + rank_spread = excluded.rank_spread, + votes = excluded.votes, + price = excluded.price, + context = excluded.context, + fetched_at = excluded.fetched_at + """, ( + m.get("rank"), raw_m, _norm(m_only), _norm(clean), + m.get("provider"), m.get("score"), int(bool(m.get("preliminary"))), + m.get("rank_spread"), m.get("votes"), m.get("price"), m.get("context"), now + )) + except Exception as e: + print(f"[ingest] text leaderboard error: {e}") + + if os.path.exists(ARENA_CACHE_AGENT): + try: + with open(ARENA_CACHE_AGENT) as f: + adata = json.load(f) + for m in adata.get("models", []): + raw_m = m["model"] + m_only = raw_m.split(" · ")[0] + m_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", m_only, flags=re.I) + clean = re.sub(r"\s*\([^)]+\)", "", m_only) + + db.execute(""" + INSERT INTO arena_agent_leaderboard ( + rank, model_name, norm_name, clean_name, provider, net_improvement, + confirmed_success, praise_vs_complaint, steerability, bash_recovery, + tool_hallucination, sessions, cost_per_task, output_tokens_per_task, price, fetched_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(model_name) DO UPDATE SET + rank = excluded.rank, + norm_name = excluded.norm_name, + clean_name = excluded.clean_name, + provider = excluded.provider, + net_improvement = excluded.net_improvement, + confirmed_success = excluded.confirmed_success, + praise_vs_complaint = excluded.praise_vs_complaint, + steerability = excluded.steerability, + bash_recovery = excluded.bash_recovery, + tool_hallucination = excluded.tool_hallucination, + sessions = excluded.sessions, + cost_per_task = excluded.cost_per_task, + output_tokens_per_task = excluded.output_tokens_per_task, + price = excluded.price, + fetched_at = excluded.fetched_at + """, ( + m.get("rank"), raw_m, _norm(m_only), _norm(clean), + m.get("provider"), m.get("net_improvement"), m.get("confirmed_success"), + m.get("praise_vs_complaint"), m.get("steerability"), m.get("bash_recovery"), + m.get("tool_hallucination"), m.get("sessions"), m.get("cost_per_task"), + m.get("output_tokens_per_task"), m.get("price"), now + )) + except Exception as e: + print(f"[ingest] agent leaderboard error: {e}") + + db.commit() # ─── OmniRoute helpers ──────────────────────────────────────────────────────── @@ -153,17 +261,39 @@ def arena_match(model_id: str, arena: dict[str, dict]) -> Optional[dict]: # ─── model sync ────────────────────────────────────────────────────────────── def sync_models(): - """Обновить список моделей в DB из OmniRoute + арены.""" + """Обновить список моделей в DB из OmniRoute + найти связи с таблицами арены.""" + db = get_db() + ingest_arena_tables(db) + try: omni = fetch_omni_models() except Exception as e: print(f"[sync] OmniRoute error: {e}") + db.close() return - arena_text = load_arena_cache(ARENA_CACHE_TEXT) - arena_agent = load_arena_cache(ARENA_CACHE_AGENT) + # Загружаем нормализованные имена из таблиц арены для сопоставления + text_rows = db.execute("SELECT model_name, norm_name, clean_name FROM arena_text_leaderboard").fetchall() + agent_rows = db.execute("SELECT model_name, norm_name, clean_name FROM arena_agent_leaderboard").fetchall() + + def find_match(base: str, cleaned: str, rows) -> Optional[str]: + bn = _norm(base) + for r in rows: + if r["norm_name"] == bn or r["clean_name"] == bn: + return r["model_name"] + cn = _norm(cleaned) + for r in rows: + if r["clean_name"] == cn: + return r["model_name"] + for candidate in (base, cleaned): + gk = _norm_keep(candidate) + if len(gk) >= 10 and (re.search(r"\d", gk) or "-" in gk): + for r in rows: + am = _norm_keep(r["model_name"]) + if gk == am or am.endswith("-" + gk) or ("-" + gk) in am or (gk in am and len(gk) >= 12): + return r["model_name"] + return None - db = get_db() now = datetime.now(timezone.utc).isoformat() for m in omni: @@ -175,36 +305,31 @@ def sync_models(): continue provider = mid.split("/")[0] - name = mid + raw = mid.split("/")[-1].lower() + base = re.sub(r"-(it|preview)(:[^:]*)?$", "", raw) + base = re.sub(r"(:[^:]*)$", "", base) + cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", base) + cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", cleaned) - ta = arena_match(mid, arena_text) - ga = arena_match(mid, arena_agent) - - # у agent-борда главная метрика — net_improvement, а не score - agent_score = ga.get("net_improvement") or ga.get("score") if ga else None + text_match = find_match(base, cleaned, text_rows) + agent_match = find_match(base, cleaned, agent_rows) db.execute(""" - INSERT INTO models (id, provider, name, arena_text_score, arena_text_rank, - arena_agent_score, arena_agent_rank, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO models (id, provider, name, raw_base, norm_base, clean_base, + text_model_match, agent_model_match, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET - arena_text_score = excluded.arena_text_score, - arena_text_rank = excluded.arena_text_rank, - arena_agent_score = excluded.arena_agent_score, - arena_agent_rank = excluded.arena_agent_rank, + text_model_match = excluded.text_model_match, + agent_model_match = excluded.agent_model_match, updated_at = excluded.updated_at """, ( - mid, provider, name, - ta["score"] if ta else None, - ta["rank"] if ta else None, - agent_score, - ga["rank"] if ga else None, - now, + mid, provider, mid, base, _norm(base), _norm(cleaned), + text_match, agent_match, now )) db.commit() db.close() - print(f"[sync] synced {len(omni)} models at {now}") + print(f"[sync] synced {len(omni)} models at {now} with SQL foreign keys") # ─── Stats helpers ──────────────────────────────────────────────────────────── @@ -257,13 +382,29 @@ def api_models( sort: str = Query("arena_text_rank"), ): db = get_db() - q = "SELECT * FROM models WHERE 1=1" + # SQL JOIN таблиц арены с моделями OmniRoute! + q = """ + SELECT + m.id, + m.provider, + m.name, + t.score as arena_text_score, + t.rank as arena_text_rank, + a.net_improvement as arena_agent_score, + a.rank as arena_agent_rank, + t.votes as arena_text_votes, + t.price as arena_text_price + FROM models m + LEFT JOIN arena_text_leaderboard t ON m.text_model_match = t.model_name + LEFT JOIN arena_agent_leaderboard a ON m.agent_model_match = a.model_name + WHERE 1=1 + """ params: list = [] if provider: - q += " AND provider = ?" + q += " AND m.provider = ?" params.append(provider) if arena_only: - q += " AND arena_text_score IS NOT NULL" + q += " AND t.score IS NOT NULL" rows = db.execute(q, params).fetchall() @@ -279,6 +420,8 @@ def api_models( "arena_text_rank": r["arena_text_rank"], "arena_agent_score": r["arena_agent_score"], "arena_agent_rank": r["arena_agent_rank"], + "arena_text_votes": r["arena_text_votes"], + "arena_text_price": r["arena_text_price"], **stats, })