fix: count reasoning-only liveness responses as healthy
Build and deploy / build-deploy (push) Waiting to run

This commit is contained in:
2026-10-27 12:22:16 +00:00
parent e8177c1270
commit 2fc17ef40b
3 changed files with 35 additions and 18 deletions
+4 -3
View File
@@ -654,10 +654,11 @@ def api_check(model_id: str):
# упадёт, реально потраченные токены сохраняются в checks. # упадёт, реально потраченные токены сохраняются в checks.
usage = extract_usage(resp) usage = extract_usage(resp)
choice = resp["choices"][0] choice = resp["choices"][0]
content = choice["message"].get("content") message = choice["message"]
ok = isinstance(content, str) and bool(content.strip()) ok = any(isinstance(message.get(field), str) and message[field].strip()
for field in ("content", "reasoning_content"))
if not ok: if not ok:
error = f"Empty text response (finish_reason={choice.get('finish_reason')})" error = f"Empty content and reasoning (finish_reason={choice.get('finish_reason')})"
except Exception as e: except Exception as e:
error = f"{type(e).__name__}: {e}" error = f"{type(e).__name__}: {e}"
print(f"[check] {model_id}: {error}") print(f"[check] {model_id}: {error}")
+8 -5
View File
@@ -51,13 +51,16 @@ def check_model(model_id: str) -> tuple[bool, int | None, str | None, dict]:
resp = json.loads(r.read()) resp = json.loads(r.read())
# Usage — до любой валидации контента. # Usage — до любой валидации контента.
usage = extract_usage(resp) usage = extract_usage(resp)
# HTTP 200 может содержать только reasoning и content=null. # HTTP 200 может содержать только reasoning при исчерпании лимита.
# Это не успешный текстовый ответ и не ошибка Python. # Наличие текста в любом из двух полей достаточно для liveness.
choice = resp["choices"][0] choice = resp["choices"][0]
content = choice["message"].get("content") message = choice["message"]
ok = isinstance(content, str) and bool(content.strip()) # Liveness: непустой текст в content ИЛИ reasoning_content.
# Usage само по себе не доказывает, что модель выдала ответ.
ok = any(isinstance(message.get(field), str) and message[field].strip()
for field in ("content", "reasoning_content"))
if not ok: if not ok:
error = f"Empty text response (finish_reason={choice.get('finish_reason')})" error = f"Empty content and reasoning (finish_reason={choice.get('finish_reason')})"
except Exception as e: except Exception as e:
error = f"{type(e).__name__}: {e}" error = f"{type(e).__name__}: {e}"
print(f" ✗ {model_id}: {error}", file=sys.stderr) print(f" ✗ {model_id}: {error}", file=sys.stderr)
+23 -10
View File
@@ -1,34 +1,47 @@
"""Оба пути пробы запрашивают none и не падают на reasoning-only ответе.""" """Оба пути пробы принимают текст в content или reasoning_content."""
import json
import sqlite3 import sqlite3
def test_cron_disables_reasoning_and_reports_null_content(core, omni_server): def test_cron_accepts_reasoning_only(core, omni_server):
omni_server.set_completions(content=None, reasoning_content="thinking", finish_reason="length", omni_server.set_completions(content=None, reasoning_content=" thinking ", finish_reason="length",
usage={"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17}) usage={"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17})
ok, latency, error, usage = core.probe.check_model("zai/test-model-a") ok, latency, error, usage = core.probe.check_model("zai/test-model-a")
assert ok is False assert ok is True and error is None
assert error == "Empty text response (finish_reason=length)"
assert usage["total_tokens"] == 17 assert usage["total_tokens"] == 17
request = omni_server.requests[-1]["body"] request = omni_server.requests[-1]["body"]
assert request["reasoning_effort"] == "none" assert request["reasoning_effort"] == "none"
assert request["max_tokens"] == 5 assert request["max_tokens"] == 5
def test_manual_disables_reasoning_and_persists_empty_text(core, omni_server): def test_manual_accepts_reasoning_only_and_persists_usage(core, omni_server):
omni_server.set_completions(content=None, reasoning_content="thinking", finish_reason="length", omni_server.set_completions(content=None, reasoning_content="thinking", finish_reason="length",
usage={"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17}) usage={"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17})
response = core.client.get("/api/check/zai/test-model-a") response = core.client.get("/api/check/zai/test-model-a")
assert response.status_code == 200 assert response.status_code == 200
assert response.json()["ok"] is False assert response.json()["ok"] is True
assert omni_server.requests[-1]["body"]["reasoning_effort"] == "none" assert omni_server.requests[-1]["body"]["reasoning_effort"] == "none"
db = sqlite3.connect(core.db_path) db = sqlite3.connect(core.db_path)
row = db.execute("SELECT ok,error,total_tokens FROM checks ORDER BY id DESC LIMIT 1").fetchone() row = db.execute("SELECT ok,error,total_tokens FROM checks ORDER BY id DESC LIMIT 1").fetchone()
db.close() db.close()
assert row == (0, "Empty text response (finish_reason=length)", 17) assert row == (1, None, 17)
def test_both_fields_empty_are_not_success(core, omni_server):
omni_server.set_completions(content=" ", reasoning_content="\t", finish_reason="length",
usage={"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17})
ok, _, error, usage = core.probe.check_model("zai/test-model-a")
assert ok is False
assert error == "Empty content and reasoning (finish_reason=length)"
assert usage["total_tokens"] == 17
response = core.client.get("/api/check/zai/test-model-a")
assert response.json()["ok"] is False
db = sqlite3.connect(core.db_path)
row = db.execute("SELECT ok,error,total_tokens FROM checks ORDER BY id DESC LIMIT 1").fetchone()
db.close()
assert row == (0, "Empty content and reasoning (finish_reason=length)", 17)
def test_text_response_still_succeeds(core, omni_server): def test_text_response_still_succeeds(core, omni_server):
omni_server.set_completions(content=" 4 ") omni_server.set_completions(content=" 4 ", reasoning_content=None)
assert core.probe.check_model("zai/test-model-a")[0] is True assert core.probe.check_model("zai/test-model-a")[0] is True
assert core.client.get("/api/check/zai/test-model-a").json()["ok"] is True assert core.client.get("/api/check/zai/test-model-a").json()["ok"] is True