fix: agent arena matching and score extraction (net_improvement)
This commit is contained in:
+17
-5
@@ -119,7 +119,11 @@ def arena_match(model_id: str, arena: dict[str, dict]) -> Optional[dict]:
|
||||
# 1) точное совпадение нормализованного имени
|
||||
bn = _norm(base)
|
||||
for k, v in arena.items():
|
||||
if _norm(k) == bn:
|
||||
# убираем вендор-хвост " Anthropic · Proprietary" из названия модели арены
|
||||
model_name_only = v.get("model", k).split(" · ")[0]
|
||||
model_name_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", model_name_only, flags=re.I)
|
||||
an = _norm(model_name_only)
|
||||
if an == bn:
|
||||
return v
|
||||
|
||||
# 2) без суффиксов тиров
|
||||
@@ -127,15 +131,20 @@ def arena_match(model_id: str, arena: dict[str, dict]) -> Optional[dict]:
|
||||
cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", cleaned)
|
||||
cn = _norm(cleaned)
|
||||
for k, v in arena.items():
|
||||
if _norm(k) == cn:
|
||||
model_name_only = v.get("model", k).split(" · ")[0]
|
||||
model_name_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", model_name_only, flags=re.I)
|
||||
# убираем скобки тиров типа (Max), (High)
|
||||
m_cleaned = re.sub(r"\s*\([^)]+\)", "", model_name_only)
|
||||
if _norm(m_cleaned) == cn:
|
||||
return v
|
||||
|
||||
# 3) containment
|
||||
for candidate in (base, cleaned):
|
||||
gk = _norm_keep(candidate)
|
||||
if len(gk) >= 12 and (re.search(r"\d", gk) or "-" in gk):
|
||||
if len(gk) >= 10 and (re.search(r"\d", gk) or "-" in gk):
|
||||
for k, v in arena.items():
|
||||
am = _norm_keep(k)
|
||||
model_name_only = v.get("model", k).split(" · ")[0]
|
||||
am = _norm_keep(model_name_only)
|
||||
if gk == am or am.endswith("-" + gk) or ("-" + gk) in am or (gk in am and len(gk) >= 12):
|
||||
return v
|
||||
return None
|
||||
@@ -171,6 +180,9 @@ def sync_models():
|
||||
ta = arena_match(mid, arena_text)
|
||||
ga = arena_match(mid, arena_agent)
|
||||
|
||||
# у agent-борда главная метрика — net_improvement, а не score
|
||||
agent_score = ga.get("net_improvement") or ga.get("score") if ga else None
|
||||
|
||||
db.execute("""
|
||||
INSERT INTO models (id, provider, name, arena_text_score, arena_text_rank,
|
||||
arena_agent_score, arena_agent_rank, updated_at)
|
||||
@@ -185,7 +197,7 @@ def sync_models():
|
||||
mid, provider, name,
|
||||
ta["score"] if ta else None,
|
||||
ta["rank"] if ta else None,
|
||||
ga["score"] if ga else None,
|
||||
agent_score,
|
||||
ga["rank"] if ga else None,
|
||||
now,
|
||||
))
|
||||
|
||||
Reference in New Issue
Block a user