#!/usr/bin/env python3
"""
omniroute_arena_report.py — прогон всех LLM на OmniRoute-гейтвее + арена-рейтинги.

Проверяет каждую openrouter/zai/nvidia/... модель гейтвея мини-чатом (2+2),
затем дополняет ответ рейтингом из arena.ai (через arena_leaderboard).
Отчёт: markdown в stdout.

Запуск:  python3 omniroute_arena_report.py [--provider openrouter] [--timeout 60]
"""

import argparse
import json
import re
import sys
import time
import urllib.error
import urllib.request

sys.path.insert(0, "/opt/data/skills/research/arena-leaderboard-parser")
from arena_leaderboard import get_leaderboard, search_models

GW = "http://192.168.1.101:20128"


def gateway_models() -> dict:
    """Все модели гейтвея по префиксам провайдеров."""
    headers = {}
    env_path = "/opt/data/.env"
    try:
        with open(env_path) as f:
            for line in f:
                if line.startswith("OMNIROUTE_API_KEY="):
                    key = line.strip().split("=", 1)[1]
                    if key:
                        headers["Authorization"] = f"Bearer {key}"
                    break
    except Exception:
        pass

    req = urllib.request.Request(f"{GW}/v1/models", headers=headers)
    data = json.loads(urllib.request.urlopen(req, timeout=15).read())
    out = {}
    for m in data["data"]:
        mid = m["id"] if isinstance(m, dict) else str(m)
        prov = mid.split("/", 1)[0]
        out.setdefault(prov, []).append(mid)
    return out


def chat(model: str, prompt="What is 2+2? Answer with just the number.",
         timeout: int = 60) -> tuple:
    """Мини-вызов. Возвращает (статус, ответ, сек). stream=True всегда (NIM-quirk)."""
    body = json.dumps({"model": model,
                       "messages": [{"role": "user", "content": prompt}],
                       "stream": True, "max_tokens": 150}).encode()
    headers = {"Content-Type": "application/json"}
    try:
        with open("/opt/data/.env") as f:
            for line in f:
                if line.startswith("OMNIROUTE_API_KEY="):
                    key = line.strip().split("=", 1)[1]
                    if key:
                        headers["Authorization"] = f"Bearer {key}"
                    break
    except Exception:
        pass

    req = urllib.request.Request(f"{GW}/v1/chat/completions", data=body,
                                 headers=headers)
    t0 = time.time()
    try:
        raw = urllib.request.urlopen(req, timeout=timeout).read().decode("utf-8", errors="replace")
        dt = time.time() - t0
        content = ""
        if raw.lstrip().startswith("data:"):
            for line in raw.splitlines():
                if line.startswith("data:") and "[DONE]" not in line:
                    try:
                        j = json.loads(line[5:].strip())
                        content += j.get("choices", [{}])[0].get("delta", {}).get("content") or ""
                    except Exception:
                        pass
        else:
            j = json.loads(raw)
            content = j.get("choices", [{}])[0].get("message", {}).get("content") or ""
        return ("ok" if content.strip() else "empty", content.strip()[:70], round(dt, 1))
    except urllib.error.HTTPError as e:
        return (f"HTTP {e.code}", e.read().decode(errors="replace")[:60], round(time.time() - t0, 1))
    except Exception as e:
        return ("fail", type(e).__name__, round(time.time() - t0, 1))


def test_with_retry(model: str, attempts: int = 2, timeout: int = 60) -> tuple:
    """1 ретрай для transient (429/5xx/empty/fail), не для вечного 403/404."""
    st, ans, dt = chat(model, timeout=timeout)
    transient = st in ("empty", "fail") or st in ("HTTP 429", "HTTP 502", "HTTP 503")
    if st != "ok" and transient and attempts > 1:
        time.sleep(2)
        st2, ans2, dt2 = chat(model, timeout=timeout)
        return st2, ans2, dt2
    return st, ans, dt


# ---- сопоставление с ареной ------------------------------------------------

def norm(s: str) -> str:
    """Норм-имя: последнее слово модели, только [a-z0-9]."""
    s = s.lower()
    s = s.split("/")[-1]
    s = re.sub(r":[^:]*$", "", s)
    s = re.sub(r"[^a-z0-9]", "", s)
    return s


def norm_keep(s: str) -> str:
    """Норм-имя с сохранением дефисов (для containment-сопоставления)."""
    s = s.lower()
    s = s.split("/")[-1]
    s = re.sub(r":[^:]*$", "", s)
    return s


def arena_lookup(gw_name: str, arena_models: list) -> tuple:
    """(arena_dict|None, note). Сопоставление гейтвей-имени с ареной."""
    # 0) Снимаем префикс провайдера (agy/..., openrouter/..., etc.)
    raw = gw_name.split("/", 1)[-1].lower()

    # Снимаем суффиксы -it, -preview, reasoning суффиксы (-thinking, -tiered, -low, -medium, -high, -disabled, -mini, -ultra, -max, -xhigh)
    base = re.sub(r"-(it|preview)(:[^:]*)?$", "", raw)
    base = re.sub(r"(:[^:]*)$", "", base)

    n_full = norm(raw)
    n = norm(base)

    # 1) точное по норм-имени
    for m in arena_models:
        if norm(m["model"]) == n_full:
            return m, "exact"

    # 2) без суффиксов (-it, -preview)
    for m in arena_models:
        if norm(m["model"]) == n:
            return m, "suffix"

    # 2.5) отсекаем reasoning / variant суффиксы: -thinking, -low, -medium, -high, -tiered, -disabled
    cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", base)
    cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", cleaned)

    # 2.6) пробуем восстановить суффиксы для точного матча (например claude-opus-4-6-thinking-high -> Claude Opus 4.6 (High))
    tier = ""
    for t in ("high", "medium", "low", "max", "xhigh"):
        if raw.endswith("-" + t):
            tier = t
            break
    if tier:
        target_name = f"{cleaned}-{tier}"
        for m in arena_models:
            if norm(m["model"]) == norm(target_name):
                return m, "variant-tier"

    n_cleaned = norm(cleaned)
    for m in arena_models:
        if norm(m["model"]) == n_cleaned:
            return m, "variant"

    # 3) containment: гейтвей-имя (с дефисами) — подстрока/суффикс арен-имени.
    #    Покрывает вендор-префиксы арены: nemotron-3-super → nvidia-nemotron-3-super.
    #    Защита от ложняков (ling→inkling): минимум 10 символов И есть цифра или дефис.
    gk = norm_keep(cleaned)
    if len(gk) >= 10 and (re.search(r"\d", gk) or "-" in gk):
        for m in arena_models:
            am = norm_keep(m["model"])
            if gk == am or am.endswith("-" + gk) or ("-" + gk) in am:
                return m, "containment"
    return None, ""


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--provider", default="openrouter",
                    help="префикс провайдера гейтвея (openrouter/zai/nvidia/...)")
    ap.add_argument("--timeout", type=int, default=60)
    args = ap.parse_args()

    models = gateway_models().get(args.provider, [])
    if not models:
        print(f"Провайдер '{args.provider}' не найден. Есть: {', '.join(gateway_models())}")
        sys.exit(1)

    lb = get_leaderboard("text")
    arena_models = lb["models"]

    print(f"# OmniRoute прогон: {args.provider} ({len(models)} моделей)\n")
    print(f"Арена-снапшот: {lb['updated']}, {lb['total_models']} моделей\n")
    print("| Модель гейтвея | Статус | Время | Arena Rank | Arena Score | Arena Model |")
    print("|---|---|---|---|---|---|")

    ok_n = 0
    for m in sorted(models):
        if any(x in m for x in ("image", "flux", "embedding")):
            st, ans, dt = "skip", "image/embed", "-"
        else:
            st, ans, dt = test_with_retry(m, timeout=args.timeout)
        ok_n += st == "ok"

        a, note = (None, "")
        if st == "ok":
            a, note = arena_lookup(m, arena_models)
        a_cell = f"#{a['rank']} ({a['score']})" if a else "—"
        a_name = a["model"] if a else "нет в арене"
        print(f"| {m} | {st} | {dt}s | {a_cell} | {a['score'] if a else '—'} | {a_name} |")

    print(f"\n**Итого: {ok_n}/{len(models)} отвечают.**")


if __name__ == "__main__":
    main()