ai_coder/arena-leaderboard-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: initial commit of arena-leaderboard-parser & omniroute benchmark tools

Browse Source
This commit is contained in:
ai_coder
2026-10-16 12:05:02 +00:00
commit f81b866e34
5 changed files with 818 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
omniroute_arena_report.py
+218
View File
@@ -0,0 +1,218 @@
#!/usr/bin/env python3
"""
omniroute_arena_report.py — прогон всех LLM на OmniRoute-гейтвее + арена-рейтинги.
Проверяет каждую openrouter/zai/nvidia/... модель гейтвея мини-чатом (2+2),
затем дополняет ответ рейтингом из arena.ai (через arena_leaderboard).
Отчёт: markdown в stdout.
Запуск: python3 omniroute_arena_report.py [--provider openrouter] [--timeout 60]
"""
import argparse
import json
import re
import sys
import time
import urllib.error
import urllib.request
sys.path.insert(0, "/opt/data/skills/research/arena-leaderboard-parser")
from arena_leaderboard import get_leaderboard, search_models
GW = "http://192.168.1.101:20128"
def gateway_models() -> dict:
"""Все модели гейтвея по префиксам провайдеров."""
headers = {}
env_path = "/opt/data/.env"
try:
with open(env_path) as f:
for line in f:
if line.startswith("OMNIROUTE_API_KEY="):
key = line.strip().split("=", 1)[1]
if key:
headers["Authorization"] = f"Bearer {key}"
break
except Exception:
pass
req = urllib.request.Request(f"{GW}/v1/models", headers=headers)
data = json.loads(urllib.request.urlopen(req, timeout=15).read())
out = {}
for m in data["data"]:
mid = m["id"] if isinstance(m, dict) else str(m)
prov = mid.split("/", 1)[0]
out.setdefault(prov, []).append(mid)
return out
def chat(model: str, prompt="What is 2+2? Answer with just the number.",
timeout: int = 60) -> tuple:
"""Мини-вызов. Возвращает (статус, ответ, сек). stream=True всегда (NIM-quirk)."""
body = json.dumps({"model": model,
"messages": [{"role": "user", "content": prompt}],
"stream": True, "max_tokens": 150}).encode()
headers = {"Content-Type": "application/json"}
try:
with open("/opt/data/.env") as f:
for line in f:
if line.startswith("OMNIROUTE_API_KEY="):
key = line.strip().split("=", 1)[1]
if key:
headers["Authorization"] = f"Bearer {key}"
break
except Exception:
pass
req = urllib.request.Request(f"{GW}/v1/chat/completions", data=body,
headers=headers)
t0 = time.time()
try:
raw = urllib.request.urlopen(req, timeout=timeout).read().decode("utf-8", errors="replace")
dt = time.time() - t0
content = ""
if raw.lstrip().startswith("data:"):
for line in raw.splitlines():
if line.startswith("data:") and "[DONE]" not in line:
try:
j = json.loads(line[5:].strip())
content += j.get("choices", [{}])[0].get("delta", {}).get("content") or ""
except Exception:
pass
else:
j = json.loads(raw)
content = j.get("choices", [{}])[0].get("message", {}).get("content") or ""
return ("ok" if content.strip() else "empty", content.strip()[:70], round(dt, 1))
except urllib.error.HTTPError as e:
return (f"HTTP {e.code}", e.read().decode(errors="replace")[:60], round(time.time() - t0, 1))
except Exception as e:
return ("fail", type(e).__name__, round(time.time() - t0, 1))
def test_with_retry(model: str, attempts: int = 2, timeout: int = 60) -> tuple:
"""1 ретрай для transient (429/5xx/empty/fail), не для вечного 403/404."""
st, ans, dt = chat(model, timeout=timeout)
transient = st in ("empty", "fail") or st in ("HTTP 429", "HTTP 502", "HTTP 503")
if st != "ok" and transient and attempts > 1:
time.sleep(2)
st2, ans2, dt2 = chat(model, timeout=timeout)
return st2, ans2, dt2
return st, ans, dt
# ---- сопоставление с ареной ------------------------------------------------
def norm(s: str) -> str:
"""Норм-имя: последнее слово модели, только [a-z0-9]."""
s = s.lower()
s = s.split("/")[-1]
s = re.sub(r":[^:]*$", "", s)
s = re.sub(r"[^a-z0-9]", "", s)
return s
def norm_keep(s: str) -> str:
"""Норм-имя с сохранением дефисов (для containment-сопоставления)."""
s = s.lower()
s = s.split("/")[-1]
s = re.sub(r":[^:]*$", "", s)
return s
def arena_lookup(gw_name: str, arena_models: list) -> tuple:
"""(arena_dict|None, note). Сопоставление гейтвей-имени с ареной."""
# 0) Снимаем префикс провайдера (agy/..., openrouter/..., etc.)
raw = gw_name.split("/", 1)[-1].lower()
# Снимаем суффиксы -it, -preview, reasoning суффиксы (-thinking, -tiered, -low, -medium, -high, -disabled, -mini, -ultra, -max, -xhigh)
base = re.sub(r"-(it|preview)(:[^:]*)?$", "", raw)
base = re.sub(r"(:[^:]*)$", "", base)
n_full = norm(raw)
n = norm(base)
# 1) точное по норм-имени
for m in arena_models:
if norm(m["model"]) == n_full:
return m, "exact"
# 2) без суффиксов (-it, -preview)
for m in arena_models:
if norm(m["model"]) == n:
return m, "suffix"
# 2.5) отсекаем reasoning / variant суффиксы: -thinking, -low, -medium, -high, -tiered, -disabled
cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", base)
cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", cleaned)
# 2.6) пробуем восстановить суффиксы для точного матча (например claude-opus-4-6-thinking-high -> Claude Opus 4.6 (High))
tier = ""
for t in ("high", "medium", "low", "max", "xhigh"):
if raw.endswith("-" + t):
tier = t
break
if tier:
target_name = f"{cleaned}-{tier}"
for m in arena_models:
if norm(m["model"]) == norm(target_name):
return m, "variant-tier"
n_cleaned = norm(cleaned)
for m in arena_models:
if norm(m["model"]) == n_cleaned:
return m, "variant"
# 3) containment: гейтвей-имя (с дефисами) — подстрока/суффикс арен-имени.
# Покрывает вендор-префиксы арены: nemotron-3-super → nvidia-nemotron-3-super.
# Защита от ложняков (ling→inkling): минимум 10 символов И есть цифра или дефис.
gk = norm_keep(cleaned)
if len(gk) >= 10 and (re.search(r"\d", gk) or "-" in gk):
for m in arena_models:
am = norm_keep(m["model"])
if gk == am or am.endswith("-" + gk) or ("-" + gk) in am:
return m, "containment"
return None, ""
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--provider", default="openrouter",
help="префикс провайдера гейтвея (openrouter/zai/nvidia/...)")
ap.add_argument("--timeout", type=int, default=60)
args = ap.parse_args()
models = gateway_models().get(args.provider, [])
if not models:
print(f"Провайдер '{args.provider}' не найден. Есть: {', '.join(gateway_models())}")
sys.exit(1)
lb = get_leaderboard("text")
arena_models = lb["models"]
print(f"# OmniRoute прогон: {args.provider} ({len(models)} моделей)\n")
print(f"Арена-снапшот: {lb['updated']}, {lb['total_models']} моделей\n")
print("| Модель гейтвея | Статус | Время | Arena Rank | Arena Score | Arena Model |")
print("|---|---|---|---|---|---|")
ok_n = 0
for m in sorted(models):
if any(x in m for x in ("image", "flux", "embedding")):
st, ans, dt = "skip", "image/embed", "-"
else:
st, ans, dt = test_with_retry(m, timeout=args.timeout)
ok_n += st == "ok"
a, note = (None, "")
if st == "ok":
a, note = arena_lookup(m, arena_models)
a_cell = f"#{a['rank']} ({a['score']})" if a else "—"
a_name = a["model"] if a else "нет в арене"
print(f"| {m} | {st} | {dt}s | {a_cell} | {a['score'] if a else '—'} | {a_name} |")
print(f"\n**Итого: {ok_n}/{len(models)} отвечают.**")
if __name__ == "__main__":
main()