219 lines
8.6 KiB
Python
219 lines
8.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
omniroute_arena_report.py — прогон всех LLM на OmniRoute-гейтвее + арена-рейтинги.
|
|
|
|
Проверяет каждую openrouter/zai/nvidia/... модель гейтвея мини-чатом (2+2),
|
|
затем дополняет ответ рейтингом из arena.ai (через arena_leaderboard).
|
|
Отчёт: markdown в stdout.
|
|
|
|
Запуск: python3 omniroute_arena_report.py [--provider openrouter] [--timeout 60]
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
sys.path.insert(0, "/opt/data/skills/research/arena-leaderboard-parser")
|
|
from arena_leaderboard import get_leaderboard, search_models
|
|
|
|
GW = "http://192.168.1.101:20128"
|
|
|
|
|
|
def gateway_models() -> dict:
|
|
"""Все модели гейтвея по префиксам провайдеров."""
|
|
headers = {}
|
|
env_path = "/opt/data/.env"
|
|
try:
|
|
with open(env_path) as f:
|
|
for line in f:
|
|
if line.startswith("OMNIROUTE_API_KEY="):
|
|
key = line.strip().split("=", 1)[1]
|
|
if key:
|
|
headers["Authorization"] = f"Bearer {key}"
|
|
break
|
|
except Exception:
|
|
pass
|
|
|
|
req = urllib.request.Request(f"{GW}/v1/models", headers=headers)
|
|
data = json.loads(urllib.request.urlopen(req, timeout=15).read())
|
|
out = {}
|
|
for m in data["data"]:
|
|
mid = m["id"] if isinstance(m, dict) else str(m)
|
|
prov = mid.split("/", 1)[0]
|
|
out.setdefault(prov, []).append(mid)
|
|
return out
|
|
|
|
|
|
def chat(model: str, prompt="What is 2+2? Answer with just the number.",
|
|
timeout: int = 60) -> tuple:
|
|
"""Мини-вызов. Возвращает (статус, ответ, сек). stream=True всегда (NIM-quirk)."""
|
|
body = json.dumps({"model": model,
|
|
"messages": [{"role": "user", "content": prompt}],
|
|
"stream": True, "max_tokens": 150}).encode()
|
|
headers = {"Content-Type": "application/json"}
|
|
try:
|
|
with open("/opt/data/.env") as f:
|
|
for line in f:
|
|
if line.startswith("OMNIROUTE_API_KEY="):
|
|
key = line.strip().split("=", 1)[1]
|
|
if key:
|
|
headers["Authorization"] = f"Bearer {key}"
|
|
break
|
|
except Exception:
|
|
pass
|
|
|
|
req = urllib.request.Request(f"{GW}/v1/chat/completions", data=body,
|
|
headers=headers)
|
|
t0 = time.time()
|
|
try:
|
|
raw = urllib.request.urlopen(req, timeout=timeout).read().decode("utf-8", errors="replace")
|
|
dt = time.time() - t0
|
|
content = ""
|
|
if raw.lstrip().startswith("data:"):
|
|
for line in raw.splitlines():
|
|
if line.startswith("data:") and "[DONE]" not in line:
|
|
try:
|
|
j = json.loads(line[5:].strip())
|
|
content += j.get("choices", [{}])[0].get("delta", {}).get("content") or ""
|
|
except Exception:
|
|
pass
|
|
else:
|
|
j = json.loads(raw)
|
|
content = j.get("choices", [{}])[0].get("message", {}).get("content") or ""
|
|
return ("ok" if content.strip() else "empty", content.strip()[:70], round(dt, 1))
|
|
except urllib.error.HTTPError as e:
|
|
return (f"HTTP {e.code}", e.read().decode(errors="replace")[:60], round(time.time() - t0, 1))
|
|
except Exception as e:
|
|
return ("fail", type(e).__name__, round(time.time() - t0, 1))
|
|
|
|
|
|
def test_with_retry(model: str, attempts: int = 2, timeout: int = 60) -> tuple:
|
|
"""1 ретрай для transient (429/5xx/empty/fail), не для вечного 403/404."""
|
|
st, ans, dt = chat(model, timeout=timeout)
|
|
transient = st in ("empty", "fail") or st in ("HTTP 429", "HTTP 502", "HTTP 503")
|
|
if st != "ok" and transient and attempts > 1:
|
|
time.sleep(2)
|
|
st2, ans2, dt2 = chat(model, timeout=timeout)
|
|
return st2, ans2, dt2
|
|
return st, ans, dt
|
|
|
|
|
|
# ---- сопоставление с ареной ------------------------------------------------
|
|
|
|
def norm(s: str) -> str:
|
|
"""Норм-имя: последнее слово модели, только [a-z0-9]."""
|
|
s = s.lower()
|
|
s = s.split("/")[-1]
|
|
s = re.sub(r":[^:]*$", "", s)
|
|
s = re.sub(r"[^a-z0-9]", "", s)
|
|
return s
|
|
|
|
|
|
def norm_keep(s: str) -> str:
|
|
"""Норм-имя с сохранением дефисов (для containment-сопоставления)."""
|
|
s = s.lower()
|
|
s = s.split("/")[-1]
|
|
s = re.sub(r":[^:]*$", "", s)
|
|
return s
|
|
|
|
|
|
def arena_lookup(gw_name: str, arena_models: list) -> tuple:
|
|
"""(arena_dict|None, note). Сопоставление гейтвей-имени с ареной."""
|
|
# 0) Снимаем префикс провайдера (agy/..., openrouter/..., etc.)
|
|
raw = gw_name.split("/", 1)[-1].lower()
|
|
|
|
# Снимаем суффиксы -it, -preview, reasoning суффиксы (-thinking, -tiered, -low, -medium, -high, -disabled, -mini, -ultra, -max, -xhigh)
|
|
base = re.sub(r"-(it|preview)(:[^:]*)?$", "", raw)
|
|
base = re.sub(r"(:[^:]*)$", "", base)
|
|
|
|
n_full = norm(raw)
|
|
n = norm(base)
|
|
|
|
# 1) точное по норм-имени
|
|
for m in arena_models:
|
|
if norm(m["model"]) == n_full:
|
|
return m, "exact"
|
|
|
|
# 2) без суффиксов (-it, -preview)
|
|
for m in arena_models:
|
|
if norm(m["model"]) == n:
|
|
return m, "suffix"
|
|
|
|
# 2.5) отсекаем reasoning / variant суффиксы: -thinking, -low, -medium, -high, -tiered, -disabled
|
|
cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", base)
|
|
cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", cleaned)
|
|
|
|
# 2.6) пробуем восстановить суффиксы для точного матча (например claude-opus-4-6-thinking-high -> Claude Opus 4.6 (High))
|
|
tier = ""
|
|
for t in ("high", "medium", "low", "max", "xhigh"):
|
|
if raw.endswith("-" + t):
|
|
tier = t
|
|
break
|
|
if tier:
|
|
target_name = f"{cleaned}-{tier}"
|
|
for m in arena_models:
|
|
if norm(m["model"]) == norm(target_name):
|
|
return m, "variant-tier"
|
|
|
|
n_cleaned = norm(cleaned)
|
|
for m in arena_models:
|
|
if norm(m["model"]) == n_cleaned:
|
|
return m, "variant"
|
|
|
|
# 3) containment: гейтвей-имя (с дефисами) — подстрока/суффикс арен-имени.
|
|
# Покрывает вендор-префиксы арены: nemotron-3-super → nvidia-nemotron-3-super.
|
|
# Защита от ложняков (ling→inkling): минимум 10 символов И есть цифра или дефис.
|
|
gk = norm_keep(cleaned)
|
|
if len(gk) >= 10 and (re.search(r"\d", gk) or "-" in gk):
|
|
for m in arena_models:
|
|
am = norm_keep(m["model"])
|
|
if gk == am or am.endswith("-" + gk) or ("-" + gk) in am:
|
|
return m, "containment"
|
|
return None, ""
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--provider", default="openrouter",
|
|
help="префикс провайдера гейтвея (openrouter/zai/nvidia/...)")
|
|
ap.add_argument("--timeout", type=int, default=60)
|
|
args = ap.parse_args()
|
|
|
|
models = gateway_models().get(args.provider, [])
|
|
if not models:
|
|
print(f"Провайдер '{args.provider}' не найден. Есть: {', '.join(gateway_models())}")
|
|
sys.exit(1)
|
|
|
|
lb = get_leaderboard("text")
|
|
arena_models = lb["models"]
|
|
|
|
print(f"# OmniRoute прогон: {args.provider} ({len(models)} моделей)\n")
|
|
print(f"Арена-снапшот: {lb['updated']}, {lb['total_models']} моделей\n")
|
|
print("| Модель гейтвея | Статус | Время | Arena Rank | Arena Score | Arena Model |")
|
|
print("|---|---|---|---|---|---|")
|
|
|
|
ok_n = 0
|
|
for m in sorted(models):
|
|
if any(x in m for x in ("image", "flux", "embedding")):
|
|
st, ans, dt = "skip", "image/embed", "-"
|
|
else:
|
|
st, ans, dt = test_with_retry(m, timeout=args.timeout)
|
|
ok_n += st == "ok"
|
|
|
|
a, note = (None, "")
|
|
if st == "ok":
|
|
a, note = arena_lookup(m, arena_models)
|
|
a_cell = f"#{a['rank']} ({a['score']})" if a else "—"
|
|
a_name = a["model"] if a else "нет в арене"
|
|
print(f"| {m} | {st} | {dt}s | {a_cell} | {a['score'] if a else '—'} | {a_name} |")
|
|
|
|
print(f"\n**Итого: {ok_n}/{len(models)} отвечают.**")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|