fix: agent arena matching and score extraction (net_improvement)
This commit is contained in:
+17
-5
@@ -119,7 +119,11 @@ def arena_match(model_id: str, arena: dict[str, dict]) -> Optional[dict]:
|
|||||||
# 1) точное совпадение нормализованного имени
|
# 1) точное совпадение нормализованного имени
|
||||||
bn = _norm(base)
|
bn = _norm(base)
|
||||||
for k, v in arena.items():
|
for k, v in arena.items():
|
||||||
if _norm(k) == bn:
|
# убираем вендор-хвост " Anthropic · Proprietary" из названия модели арены
|
||||||
|
model_name_only = v.get("model", k).split(" · ")[0]
|
||||||
|
model_name_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", model_name_only, flags=re.I)
|
||||||
|
an = _norm(model_name_only)
|
||||||
|
if an == bn:
|
||||||
return v
|
return v
|
||||||
|
|
||||||
# 2) без суффиксов тиров
|
# 2) без суффиксов тиров
|
||||||
@@ -127,15 +131,20 @@ def arena_match(model_id: str, arena: dict[str, dict]) -> Optional[dict]:
|
|||||||
cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", cleaned)
|
cleaned = re.sub(r"-(thinking|tiered|disabled|low|medium|high|xhigh|max|ultra|mini)", "", cleaned)
|
||||||
cn = _norm(cleaned)
|
cn = _norm(cleaned)
|
||||||
for k, v in arena.items():
|
for k, v in arena.items():
|
||||||
if _norm(k) == cn:
|
model_name_only = v.get("model", k).split(" · ")[0]
|
||||||
|
model_name_only = re.sub(r"\s+(Google|Anthropic|OpenAI|Meta|Nvidia|Mistral|DeepSeek|Alibaba)$", "", model_name_only, flags=re.I)
|
||||||
|
# убираем скобки тиров типа (Max), (High)
|
||||||
|
m_cleaned = re.sub(r"\s*\([^)]+\)", "", model_name_only)
|
||||||
|
if _norm(m_cleaned) == cn:
|
||||||
return v
|
return v
|
||||||
|
|
||||||
# 3) containment
|
# 3) containment
|
||||||
for candidate in (base, cleaned):
|
for candidate in (base, cleaned):
|
||||||
gk = _norm_keep(candidate)
|
gk = _norm_keep(candidate)
|
||||||
if len(gk) >= 12 and (re.search(r"\d", gk) or "-" in gk):
|
if len(gk) >= 10 and (re.search(r"\d", gk) or "-" in gk):
|
||||||
for k, v in arena.items():
|
for k, v in arena.items():
|
||||||
am = _norm_keep(k)
|
model_name_only = v.get("model", k).split(" · ")[0]
|
||||||
|
am = _norm_keep(model_name_only)
|
||||||
if gk == am or am.endswith("-" + gk) or ("-" + gk) in am or (gk in am and len(gk) >= 12):
|
if gk == am or am.endswith("-" + gk) or ("-" + gk) in am or (gk in am and len(gk) >= 12):
|
||||||
return v
|
return v
|
||||||
return None
|
return None
|
||||||
@@ -171,6 +180,9 @@ def sync_models():
|
|||||||
ta = arena_match(mid, arena_text)
|
ta = arena_match(mid, arena_text)
|
||||||
ga = arena_match(mid, arena_agent)
|
ga = arena_match(mid, arena_agent)
|
||||||
|
|
||||||
|
# у agent-борда главная метрика — net_improvement, а не score
|
||||||
|
agent_score = ga.get("net_improvement") or ga.get("score") if ga else None
|
||||||
|
|
||||||
db.execute("""
|
db.execute("""
|
||||||
INSERT INTO models (id, provider, name, arena_text_score, arena_text_rank,
|
INSERT INTO models (id, provider, name, arena_text_score, arena_text_rank,
|
||||||
arena_agent_score, arena_agent_rank, updated_at)
|
arena_agent_score, arena_agent_rank, updated_at)
|
||||||
@@ -185,7 +197,7 @@ def sync_models():
|
|||||||
mid, provider, name,
|
mid, provider, name,
|
||||||
ta["score"] if ta else None,
|
ta["score"] if ta else None,
|
||||||
ta["rank"] if ta else None,
|
ta["rank"] if ta else None,
|
||||||
ga["score"] if ga else None,
|
agent_score,
|
||||||
ga["rank"] if ga else None,
|
ga["rank"] if ga else None,
|
||||||
now,
|
now,
|
||||||
))
|
))
|
||||||
|
|||||||
Reference in New Issue
Block a user