feat: store arena leaderboards in relational SQLite tables and JOIN with models
This commit is contained in:
+188
-45
@@ -35,15 +35,52 @@ def get_db() -> sqlite3.Connection:
|
||||
def init_db():
|
||||
db = get_db()
|
||||
db.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS arena_text_leaderboard (
|
||||
rank INTEGER,
|
||||
model_name TEXT PRIMARY KEY,
|
||||
norm_name TEXT NOT NULL,
|
||||
clean_name TEXT NOT NULL,
|
||||
provider TEXT,
|
||||
score TEXT,
|
||||
preliminary INTEGER,
|
||||
rank_spread TEXT,
|
||||
votes INTEGER,
|
||||
price TEXT,
|
||||
context TEXT,
|
||||
fetched_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS arena_agent_leaderboard (
|
||||
rank INTEGER,
|
||||
model_name TEXT PRIMARY KEY,
|
||||
norm_name TEXT NOT NULL,
|
||||
clean_name TEXT NOT NULL,
|
||||
provider TEXT,
|
||||
net_improvement TEXT,
|
||||
confirmed_success TEXT,
|
||||
praise_vs_complaint TEXT,
|
||||
steerability TEXT,
|
||||
bash_recovery TEXT,
|
||||
tool_hallucination TEXT,
|
||||
sessions INTEGER,
|
||||
cost_per_task TEXT,
|
||||
output_tokens_per_task TEXT,
|
||||
price TEXT,
|
||||
fetched_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS models (
|
||||
id TEXT PRIMARY KEY,
|
||||
provider TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
arena_text_score TEXT,
|
||||
arena_text_rank INTEGER,
|
||||
arena_agent_score TEXT,
|
||||
arena_agent_rank INTEGER,
|
||||
updated_at TEXT
|
||||
raw_base TEXT NOT NULL,
|
||||
norm_base TEXT NOT NULL,
|
||||
clean_base TEXT NOT NULL,
|
||||
text_model_match TEXT,
|
||||
agent_model_match TEXT,
|
||||
updated_at TEXT,
|
||||
FOREIGN KEY(text_model_match) REFERENCES arena_text_leaderboard(model_name),
|
||||
FOREIGN KEY(agent_model_match) REFERENCES arena_agent_leaderboard(model_name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS checks (
|
||||
@@ -56,26 +93,97 @@ def init_db():
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_checks_model ON checks(model_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_checks_time ON checks(checked_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_at_norm ON arena_text_leaderboard(norm_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_ag_norm ON arena_agent_leaderboard(norm_name);
|
||||
""")
|
||||
db.commit()
|
||||
db.close()
|
||||
|
||||
|
||||
# ─── Arena data loading ──────────────────────────────────────────────────────
|
||||
# ─── Arena data ingestion into DB ───────────────────────────────────────────
|
||||
|
||||
def load_arena_cache(path: str) -> dict[str, dict]:
|
||||
"""Возвращает dict {normalized_model_name: {rank, score}} из кеш-файла арены."""
|
||||
if not os.path.exists(path):
|
||||
return {}
|
||||
def ingest_arena_tables(db: sqlite3.Connection):
|
||||
"""Импортирует сырые JSON кеши арены прямо в реляционные таблицы SQLite."""
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
if os.path.exists(ARENA_CACHE_TEXT):
|
||||
try:
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
result = {}
|
||||
for m in data.get("models", []):
|
||||
result[m["model"].lower()] = m
|
||||
return result
|
||||
except Exception:
|
||||
return {}
|
||||
with open(ARENA_CACHE_TEXT) as f:
|
||||
tdata = json.load(f)
|
||||
for m in tdata.get("models", []):
|
||||
raw_m = m["model"]
|
||||
m_only = raw_m.split(" · ")[0]
|
||||
m_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", m_only, flags=re.I)
|
||||
clean = re.sub(r"\s*\([^)]+\)", "", m_only)
|
||||
|
||||
db.execute("""
|
||||
INSERT INTO arena_text_leaderboard (
|
||||
rank, model_name, norm_name, clean_name, provider, score,
|
||||
preliminary, rank_spread, votes, price, context, fetched_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(model_name) DO UPDATE SET
|
||||
rank = excluded.rank,
|
||||
norm_name = excluded.norm_name,
|
||||
clean_name = excluded.clean_name,
|
||||
provider = excluded.provider,
|
||||
score = excluded.score,
|
||||
preliminary = excluded.preliminary,
|
||||
rank_spread = excluded.rank_spread,
|
||||
votes = excluded.votes,
|
||||
price = excluded.price,
|
||||
context = excluded.context,
|
||||
fetched_at = excluded.fetched_at
|
||||
""", (
|
||||
m.get("rank"), raw_m, _norm(m_only), _norm(clean),
|
||||
m.get("provider"), m.get("score"), int(bool(m.get("preliminary"))),
|
||||
m.get("rank_spread"), m.get("votes"), m.get("price"), m.get("context"), now
|
||||
))
|
||||
except Exception as e:
|
||||
print(f"[ingest] text leaderboard error: {e}")
|
||||
|
||||
if os.path.exists(ARENA_CACHE_AGENT):
|
||||
try:
|
||||
with open(ARENA_CACHE_AGENT) as f:
|
||||
adata = json.load(f)
|
||||
for m in adata.get("models", []):
|
||||
raw_m = m["model"]
|
||||
m_only = raw_m.split(" · ")[0]
|
||||
m_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", m_only, flags=re.I)
|
||||
clean = re.sub(r"\s*\([^)]+\)", "", m_only)
|
||||
|
||||
db.execute("""
|
||||
INSERT INTO arena_agent_leaderboard (
|
||||
rank, model_name, norm_name, clean_name, provider, net_improvement,
|
||||
confirmed_success, praise_vs_complaint, steerability, bash_recovery,
|
||||
tool_hallucination, sessions, cost_per_task, output_tokens_per_task, price, fetched_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(model_name) DO UPDATE SET
|
||||
rank = excluded.rank,
|
||||
norm_name = excluded.norm_name,
|
||||
clean_name = excluded.clean_name,
|
||||
provider = excluded.provider,
|
||||
net_improvement = excluded.net_improvement,
|
||||
confirmed_success = excluded.confirmed_success,
|
||||
praise_vs_complaint = excluded.praise_vs_complaint,
|
||||
steerability = excluded.steerability,
|
||||
bash_recovery = excluded.bash_recovery,
|
||||
tool_hallucination = excluded.tool_hallucination,
|
||||
sessions = excluded.sessions,
|
||||
cost_per_task = excluded.cost_per_task,
|
||||
output_tokens_per_task = excluded.output_tokens_per_task,
|
||||
price = excluded.price,
|
||||
fetched_at = excluded.fetched_at
|
||||
""", (
|
||||
m.get("rank"), raw_m, _norm(m_only), _norm(clean),
|
||||
m.get("provider"), m.get("net_improvement"), m.get("confirmed_success"),
|
||||
m.get("praise_vs_complaint"), m.get("steerability"), m.get("bash_recovery"),
|
||||
m.get("tool_hallucination"), m.get("sessions"), m.get("cost_per_task"),
|
||||
m.get("output_tokens_per_task"), m.get("price"), now
|
||||
))
|
||||
except Exception as e:
|
||||
print(f"[ingest] agent leaderboard error: {e}")
|
||||
|
||||
db.commit()
|
||||
|
||||
|
||||
# ─── OmniRoute helpers ────────────────────────────────────────────────────────
|
||||
@@ -153,17 +261,39 @@ def arena_match(model_id: str, arena: dict[str, dict]) -> Optional[dict]:
|
||||
# ─── model sync ──────────────────────────────────────────────────────────────
|
||||
|
||||
def sync_models():
|
||||
"""Обновить список моделей в DB из OmniRoute + арены."""
|
||||
"""Обновить список моделей в DB из OmniRoute + найти связи с таблицами арены."""
|
||||
db = get_db()
|
||||
ingest_arena_tables(db)
|
||||
|
||||
try:
|
||||
omni = fetch_omni_models()
|
||||
except Exception as e:
|
||||
print(f"[sync] OmniRoute error: {e}")
|
||||
db.close()
|
||||
return
|
||||
|
||||
arena_text = load_arena_cache(ARENA_CACHE_TEXT)
|
||||
arena_agent = load_arena_cache(ARENA_CACHE_AGENT)
|
||||
# Загружаем нормализованные имена из таблиц арены для сопоставления
|
||||
text_rows = db.execute("SELECT model_name, norm_name, clean_name FROM arena_text_leaderboard").fetchall()
|
||||
agent_rows = db.execute("SELECT model_name, norm_name, clean_name FROM arena_agent_leaderboard").fetchall()
|
||||
|
||||
def find_match(base: str, cleaned: str, rows) -> Optional[str]:
|
||||
bn = _norm(base)
|
||||
for r in rows:
|
||||
if r["norm_name"] == bn or r["clean_name"] == bn:
|
||||
return r["model_name"]
|
||||
cn = _norm(cleaned)
|
||||
for r in rows:
|
||||
if r["clean_name"] == cn:
|
||||
return r["model_name"]
|
||||
for candidate in (base, cleaned):
|
||||
gk = _norm_keep(candidate)
|
||||
if len(gk) >= 10 and (re.search(r"\d", gk) or "-" in gk):
|
||||
for r in rows:
|
||||
am = _norm_keep(r["model_name"])
|
||||
if gk == am or am.endswith("-" + gk) or ("-" + gk) in am or (gk in am and len(gk) >= 12):
|
||||
return r["model_name"]
|
||||
return None
|
||||
|
||||
db = get_db()
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
for m in omni:
|
||||
@@ -175,36 +305,31 @@ def sync_models():
|
||||
continue
|
||||
|
||||
provider = mid.split("/")[0]
|
||||
name = mid
|
||||
raw = mid.split("/")[-1].lower()
|
||||
base = re.sub(r"-(it|preview)(:[^:]*)?$", "", raw)
|
||||
base = re.sub(r"(:[^:]*)$", "", base)
|
||||
cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", base)
|
||||
cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", cleaned)
|
||||
|
||||
ta = arena_match(mid, arena_text)
|
||||
ga = arena_match(mid, arena_agent)
|
||||
|
||||
# у agent-борда главная метрика — net_improvement, а не score
|
||||
agent_score = ga.get("net_improvement") or ga.get("score") if ga else None
|
||||
text_match = find_match(base, cleaned, text_rows)
|
||||
agent_match = find_match(base, cleaned, agent_rows)
|
||||
|
||||
db.execute("""
|
||||
INSERT INTO models (id, provider, name, arena_text_score, arena_text_rank,
|
||||
arena_agent_score, arena_agent_rank, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO models (id, provider, name, raw_base, norm_base, clean_base,
|
||||
text_model_match, agent_model_match, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
arena_text_score = excluded.arena_text_score,
|
||||
arena_text_rank = excluded.arena_text_rank,
|
||||
arena_agent_score = excluded.arena_agent_score,
|
||||
arena_agent_rank = excluded.arena_agent_rank,
|
||||
text_model_match = excluded.text_model_match,
|
||||
agent_model_match = excluded.agent_model_match,
|
||||
updated_at = excluded.updated_at
|
||||
""", (
|
||||
mid, provider, name,
|
||||
ta["score"] if ta else None,
|
||||
ta["rank"] if ta else None,
|
||||
agent_score,
|
||||
ga["rank"] if ga else None,
|
||||
now,
|
||||
mid, provider, mid, base, _norm(base), _norm(cleaned),
|
||||
text_match, agent_match, now
|
||||
))
|
||||
|
||||
db.commit()
|
||||
db.close()
|
||||
print(f"[sync] synced {len(omni)} models at {now}")
|
||||
print(f"[sync] synced {len(omni)} models at {now} with SQL foreign keys")
|
||||
|
||||
|
||||
# ─── Stats helpers ────────────────────────────────────────────────────────────
|
||||
@@ -257,13 +382,29 @@ def api_models(
|
||||
sort: str = Query("arena_text_rank"),
|
||||
):
|
||||
db = get_db()
|
||||
q = "SELECT * FROM models WHERE 1=1"
|
||||
# SQL JOIN таблиц арены с моделями OmniRoute!
|
||||
q = """
|
||||
SELECT
|
||||
m.id,
|
||||
m.provider,
|
||||
m.name,
|
||||
t.score as arena_text_score,
|
||||
t.rank as arena_text_rank,
|
||||
a.net_improvement as arena_agent_score,
|
||||
a.rank as arena_agent_rank,
|
||||
t.votes as arena_text_votes,
|
||||
t.price as arena_text_price
|
||||
FROM models m
|
||||
LEFT JOIN arena_text_leaderboard t ON m.text_model_match = t.model_name
|
||||
LEFT JOIN arena_agent_leaderboard a ON m.agent_model_match = a.model_name
|
||||
WHERE 1=1
|
||||
"""
|
||||
params: list = []
|
||||
if provider:
|
||||
q += " AND provider = ?"
|
||||
q += " AND m.provider = ?"
|
||||
params.append(provider)
|
||||
if arena_only:
|
||||
q += " AND arena_text_score IS NOT NULL"
|
||||
q += " AND t.score IS NOT NULL"
|
||||
|
||||
rows = db.execute(q, params).fetchall()
|
||||
|
||||
@@ -279,6 +420,8 @@ def api_models(
|
||||
"arena_text_rank": r["arena_text_rank"],
|
||||
"arena_agent_score": r["arena_agent_score"],
|
||||
"arena_agent_rank": r["arena_agent_rank"],
|
||||
"arena_text_votes": r["arena_text_votes"],
|
||||
"arena_text_price": r["arena_text_price"],
|
||||
**stats,
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user