ai_coder/arena-dashboard
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat: store arena leaderboards in relational SQLite tables and JOIN with models

Browse Source
This commit is contained in:
ai_coder
2026-10-16 15:34:50 +00:00
parent d8fa0af825
commit 296fa9c7ad
1 changed files with 189 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/main.py
+189 -46
View File
@@ -35,15 +35,52 @@ def get_db() -> sqlite3.Connection:
def init_db():
db = get_db()
db.executescript("""
CREATE TABLE IF NOT EXISTS arena_text_leaderboard (
rank INTEGER,
model_name TEXT PRIMARY KEY,
norm_name TEXT NOT NULL,
clean_name TEXT NOT NULL,
provider TEXT,
score TEXT,
preliminary INTEGER,
rank_spread TEXT,
votes INTEGER,
price TEXT,
context TEXT,
fetched_at TEXT
);
CREATE TABLE IF NOT EXISTS arena_agent_leaderboard (
rank INTEGER,
model_name TEXT PRIMARY KEY,
norm_name TEXT NOT NULL,
clean_name TEXT NOT NULL,
provider TEXT,
net_improvement TEXT,
confirmed_success TEXT,
praise_vs_complaint TEXT,
steerability TEXT,
bash_recovery TEXT,
tool_hallucination TEXT,
sessions INTEGER,
cost_per_task TEXT,
output_tokens_per_task TEXT,
price TEXT,
fetched_at TEXT
);
CREATE TABLE IF NOT EXISTS models (
id TEXT PRIMARY KEY,
provider TEXT NOT NULL,
name TEXT NOT NULL,
arena_text_score TEXT,
arena_text_rank INTEGER,
arena_agent_score TEXT,
arena_agent_rank INTEGER,
updated_at TEXT
raw_base TEXT NOT NULL,
norm_base TEXT NOT NULL,
clean_base TEXT NOT NULL,
text_model_match TEXT,
agent_model_match TEXT,
updated_at TEXT,
FOREIGN KEY(text_model_match) REFERENCES arena_text_leaderboard(model_name),
FOREIGN KEY(agent_model_match) REFERENCES arena_agent_leaderboard(model_name)
);
CREATE TABLE IF NOT EXISTS checks (
@@ -56,26 +93,97 @@ def init_db():
CREATE INDEX IF NOT EXISTS idx_checks_model ON checks(model_id);
CREATE INDEX IF NOT EXISTS idx_checks_time ON checks(checked_at);
CREATE INDEX IF NOT EXISTS idx_at_norm ON arena_text_leaderboard(norm_name);
CREATE INDEX IF NOT EXISTS idx_ag_norm ON arena_agent_leaderboard(norm_name);
""")
db.commit()
db.close()
# ─── Arena data loading ──────────────────────────────────────────────────────
# ─── Arena data ingestion into DB ───────────────────────────────────────────
def load_arena_cache(path: str) -> dict[str, dict]:
"""Возвращает dict {normalized_model_name: {rank, score}} из кеш-файла арены."""
if not os.path.exists(path):
return {}
try:
with open(path) as f:
data = json.load(f)
result = {}
for m in data.get("models", []):
result[m["model"].lower()] = m
return result
except Exception:
return {}
def ingest_arena_tables(db: sqlite3.Connection):
"""Импортирует сырые JSON кеши арены прямо в реляционные таблицы SQLite."""
now = datetime.now(timezone.utc).isoformat()
if os.path.exists(ARENA_CACHE_TEXT):
try:
with open(ARENA_CACHE_TEXT) as f:
tdata = json.load(f)
for m in tdata.get("models", []):
raw_m = m["model"]
m_only = raw_m.split(" · ")[0]
m_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", m_only, flags=re.I)
clean = re.sub(r"\s*\([^)]+\)", "", m_only)
db.execute("""
INSERT INTO arena_text_leaderboard (
rank, model_name, norm_name, clean_name, provider, score,
preliminary, rank_spread, votes, price, context, fetched_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(model_name) DO UPDATE SET
rank = excluded.rank,
norm_name = excluded.norm_name,
clean_name = excluded.clean_name,
provider = excluded.provider,
score = excluded.score,
preliminary = excluded.preliminary,
rank_spread = excluded.rank_spread,
votes = excluded.votes,
price = excluded.price,
context = excluded.context,
fetched_at = excluded.fetched_at
""", (
m.get("rank"), raw_m, _norm(m_only), _norm(clean),
m.get("provider"), m.get("score"), int(bool(m.get("preliminary"))),
m.get("rank_spread"), m.get("votes"), m.get("price"), m.get("context"), now
))
except Exception as e:
print(f"[ingest] text leaderboard error: {e}")
if os.path.exists(ARENA_CACHE_AGENT):
try:
with open(ARENA_CACHE_AGENT) as f:
adata = json.load(f)
for m in adata.get("models", []):
raw_m = m["model"]
m_only = raw_m.split(" · ")[0]
m_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", m_only, flags=re.I)
clean = re.sub(r"\s*\([^)]+\)", "", m_only)
db.execute("""
INSERT INTO arena_agent_leaderboard (
rank, model_name, norm_name, clean_name, provider, net_improvement,
confirmed_success, praise_vs_complaint, steerability, bash_recovery,
tool_hallucination, sessions, cost_per_task, output_tokens_per_task, price, fetched_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(model_name) DO UPDATE SET
rank = excluded.rank,
norm_name = excluded.norm_name,
clean_name = excluded.clean_name,
provider = excluded.provider,
net_improvement = excluded.net_improvement,
confirmed_success = excluded.confirmed_success,
praise_vs_complaint = excluded.praise_vs_complaint,
steerability = excluded.steerability,
bash_recovery = excluded.bash_recovery,
tool_hallucination = excluded.tool_hallucination,
sessions = excluded.sessions,
cost_per_task = excluded.cost_per_task,
output_tokens_per_task = excluded.output_tokens_per_task,
price = excluded.price,
fetched_at = excluded.fetched_at
""", (
m.get("rank"), raw_m, _norm(m_only), _norm(clean),
m.get("provider"), m.get("net_improvement"), m.get("confirmed_success"),
m.get("praise_vs_complaint"), m.get("steerability"), m.get("bash_recovery"),
m.get("tool_hallucination"), m.get("sessions"), m.get("cost_per_task"),
m.get("output_tokens_per_task"), m.get("price"), now
))
except Exception as e:
print(f"[ingest] agent leaderboard error: {e}")
db.commit()
# ─── OmniRoute helpers ────────────────────────────────────────────────────────
@@ -153,17 +261,39 @@ def arena_match(model_id: str, arena: dict[str, dict]) -> Optional[dict]:
# ─── model sync ──────────────────────────────────────────────────────────────
def sync_models():
"""Обновить список моделей в DB из OmniRoute + арены."""
"""Обновить список моделей в DB из OmniRoute + найти связи с таблицами арены."""
db = get_db()
ingest_arena_tables(db)
try:
omni = fetch_omni_models()
except Exception as e:
print(f"[sync] OmniRoute error: {e}")
db.close()
return
arena_text = load_arena_cache(ARENA_CACHE_TEXT)
arena_agent = load_arena_cache(ARENA_CACHE_AGENT)
# Загружаем нормализованные имена из таблиц арены для сопоставления
text_rows = db.execute("SELECT model_name, norm_name, clean_name FROM arena_text_leaderboard").fetchall()
agent_rows = db.execute("SELECT model_name, norm_name, clean_name FROM arena_agent_leaderboard").fetchall()
def find_match(base: str, cleaned: str, rows) -> Optional[str]:
bn = _norm(base)
for r in rows:
if r["norm_name"] == bn or r["clean_name"] == bn:
return r["model_name"]
cn = _norm(cleaned)
for r in rows:
if r["clean_name"] == cn:
return r["model_name"]
for candidate in (base, cleaned):
gk = _norm_keep(candidate)
if len(gk) >= 10 and (re.search(r"\d", gk) or "-" in gk):
for r in rows:
am = _norm_keep(r["model_name"])
if gk == am or am.endswith("-" + gk) or ("-" + gk) in am or (gk in am and len(gk) >= 12):
return r["model_name"]
return None
db = get_db()
now = datetime.now(timezone.utc).isoformat()
for m in omni:
@@ -175,36 +305,31 @@ def sync_models():
continue
provider = mid.split("/")[0]
name = mid
raw = mid.split("/")[-1].lower()
base = re.sub(r"-(it|preview)(:[^:]*)?$", "", raw)
base = re.sub(r"(:[^:]*)$", "", base)
cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", base)
cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", cleaned)
ta = arena_match(mid, arena_text)
ga = arena_match(mid, arena_agent)
# у agent-борда главная метрика — net_improvement, а не score
agent_score = ga.get("net_improvement") or ga.get("score") if ga else None
text_match = find_match(base, cleaned, text_rows)
agent_match = find_match(base, cleaned, agent_rows)
db.execute("""
INSERT INTO models (id, provider, name, arena_text_score, arena_text_rank,
arena_agent_score, arena_agent_rank, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO models (id, provider, name, raw_base, norm_base, clean_base,
text_model_match, agent_model_match, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
arena_text_score = excluded.arena_text_score,
arena_text_rank = excluded.arena_text_rank,
arena_agent_score = excluded.arena_agent_score,
arena_agent_rank = excluded.arena_agent_rank,
text_model_match = excluded.text_model_match,
agent_model_match = excluded.agent_model_match,
updated_at = excluded.updated_at
""", (
mid, provider, name,
ta["score"] if ta else None,
ta["rank"] if ta else None,
agent_score,
ga["rank"] if ga else None,
now,
mid, provider, mid, base, _norm(base), _norm(cleaned),
text_match, agent_match, now
))
db.commit()
db.close()
print(f"[sync] synced {len(omni)} models at {now}")
print(f"[sync] synced {len(omni)} models at {now} with SQL foreign keys")
# ─── Stats helpers ────────────────────────────────────────────────────────────
@@ -257,13 +382,29 @@ def api_models(
sort: str = Query("arena_text_rank"),
):
db = get_db()
q = "SELECT * FROM models WHERE 1=1"
# SQL JOIN таблиц арены с моделями OmniRoute!
q = """
SELECT
m.id,
m.provider,
m.name,
t.score as arena_text_score,
t.rank as arena_text_rank,
a.net_improvement as arena_agent_score,
a.rank as arena_agent_rank,
t.votes as arena_text_votes,
t.price as arena_text_price
FROM models m
LEFT JOIN arena_text_leaderboard t ON m.text_model_match = t.model_name
LEFT JOIN arena_agent_leaderboard a ON m.agent_model_match = a.model_name
WHERE 1=1
"""
params: list = []
if provider:
q += " AND provider = ?"
q += " AND m.provider = ?"
params.append(provider)
if arena_only:
q += " AND arena_text_score IS NOT NULL"
q += " AND t.score IS NOT NULL"
rows = db.execute(q, params).fetchall()
@@ -279,6 +420,8 @@ def api_models(
"arena_text_rank": r["arena_text_rank"],
"arena_agent_score": r["arena_agent_score"],
"arena_agent_rank": r["arena_agent_rank"],
"arena_text_votes": r["arena_text_votes"],
"arena_text_price": r["arena_text_price"],
**stats,
})