fix: agent arena matching and score extraction (net_improvement)

This commit is contained in:
2026-10-16 15:16:36 +00:00
parent 8c25a1dbe1
commit d8fa0af825
+17 -5
View File
@@ -119,7 +119,11 @@ def arena_match(model_id: str, arena: dict[str, dict]) -> Optional[dict]:
# 1) точное совпадение нормализованного имени # 1) точное совпадение нормализованного имени
bn = _norm(base) bn = _norm(base)
for k, v in arena.items(): for k, v in arena.items():
if _norm(k) == bn: # убираем вендор-хвост " Anthropic · Proprietary" из названия модели арены
model_name_only = v.get("model", k).split(" · ")[0]
model_name_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", model_name_only, flags=re.I)
an = _norm(model_name_only)
if an == bn:
return v return v
# 2) без суффиксов тиров # 2) без суффиксов тиров
@@ -127,15 +131,20 @@ def arena_match(model_id: str, arena: dict[str, dict]) -> Optional[dict]:
cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", cleaned) cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", cleaned)
cn = _norm(cleaned) cn = _norm(cleaned)
for k, v in arena.items(): for k, v in arena.items():
if _norm(k) == cn: model_name_only = v.get("model", k).split(" · ")[0]
model_name_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", model_name_only, flags=re.I)
# убираем скобки тиров типа (Max), (High)
m_cleaned = re.sub(r"\s*\([^)]+\)", "", model_name_only)
if _norm(m_cleaned) == cn:
return v return v
# 3) containment # 3) containment
for candidate in (base, cleaned): for candidate in (base, cleaned):
gk = _norm_keep(candidate) gk = _norm_keep(candidate)
if len(gk) >= 12 and (re.search(r"\d", gk) or "-" in gk): if len(gk) >= 10 and (re.search(r"\d", gk) or "-" in gk):
for k, v in arena.items(): for k, v in arena.items():
am = _norm_keep(k) model_name_only = v.get("model", k).split(" · ")[0]
am = _norm_keep(model_name_only)
if gk == am or am.endswith("-" + gk) or ("-" + gk) in am or (gk in am and len(gk) >= 12): if gk == am or am.endswith("-" + gk) or ("-" + gk) in am or (gk in am and len(gk) >= 12):
return v return v
return None return None
@@ -171,6 +180,9 @@ def sync_models():
ta = arena_match(mid, arena_text) ta = arena_match(mid, arena_text)
ga = arena_match(mid, arena_agent) ga = arena_match(mid, arena_agent)
# у agent-борда главная метрика — net_improvement, а не score
agent_score = ga.get("net_improvement") or ga.get("score") if ga else None
db.execute(""" db.execute("""
INSERT INTO models (id, provider, name, arena_text_score, arena_text_rank, INSERT INTO models (id, provider, name, arena_text_score, arena_text_rank,
arena_agent_score, arena_agent_rank, updated_at) arena_agent_score, arena_agent_rank, updated_at)
@@ -185,7 +197,7 @@ def sync_models():
mid, provider, name, mid, provider, name,
ta["score"] if ta else None, ta["score"] if ta else None,
ta["rank"] if ta else None, ta["rank"] if ta else None,
ga["score"] if ga else None, agent_score,
ga["rank"] if ga else None, ga["rank"] if ga else None,
now, now,
)) ))