fix: count reasoning-only liveness responses as healthy
Build and deploy / build-deploy (push) Waiting to run
Build and deploy / build-deploy (push) Waiting to run
This commit is contained in:
+4
-3
@@ -654,10 +654,11 @@ def api_check(model_id: str):
|
|||||||
# упадёт, реально потраченные токены сохраняются в checks.
|
# упадёт, реально потраченные токены сохраняются в checks.
|
||||||
usage = extract_usage(resp)
|
usage = extract_usage(resp)
|
||||||
choice = resp["choices"][0]
|
choice = resp["choices"][0]
|
||||||
content = choice["message"].get("content")
|
message = choice["message"]
|
||||||
ok = isinstance(content, str) and bool(content.strip())
|
ok = any(isinstance(message.get(field), str) and message[field].strip()
|
||||||
|
for field in ("content", "reasoning_content"))
|
||||||
if not ok:
|
if not ok:
|
||||||
error = f"Empty text response (finish_reason={choice.get('finish_reason')})"
|
error = f"Empty content and reasoning (finish_reason={choice.get('finish_reason')})"
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
error = f"{type(e).__name__}: {e}"
|
error = f"{type(e).__name__}: {e}"
|
||||||
print(f"[check] {model_id}: {error}")
|
print(f"[check] {model_id}: {error}")
|
||||||
|
|||||||
+8
-5
@@ -51,13 +51,16 @@ def check_model(model_id: str) -> tuple[bool, int | None, str | None, dict]:
|
|||||||
resp = json.loads(r.read())
|
resp = json.loads(r.read())
|
||||||
# Usage — до любой валидации контента.
|
# Usage — до любой валидации контента.
|
||||||
usage = extract_usage(resp)
|
usage = extract_usage(resp)
|
||||||
# HTTP 200 может содержать только reasoning и content=null.
|
# HTTP 200 может содержать только reasoning при исчерпании лимита.
|
||||||
# Это не успешный текстовый ответ и не ошибка Python.
|
# Наличие текста в любом из двух полей достаточно для liveness.
|
||||||
choice = resp["choices"][0]
|
choice = resp["choices"][0]
|
||||||
content = choice["message"].get("content")
|
message = choice["message"]
|
||||||
ok = isinstance(content, str) and bool(content.strip())
|
# Liveness: непустой текст в content ИЛИ reasoning_content.
|
||||||
|
# Usage само по себе не доказывает, что модель выдала ответ.
|
||||||
|
ok = any(isinstance(message.get(field), str) and message[field].strip()
|
||||||
|
for field in ("content", "reasoning_content"))
|
||||||
if not ok:
|
if not ok:
|
||||||
error = f"Empty text response (finish_reason={choice.get('finish_reason')})"
|
error = f"Empty content and reasoning (finish_reason={choice.get('finish_reason')})"
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
error = f"{type(e).__name__}: {e}"
|
error = f"{type(e).__name__}: {e}"
|
||||||
print(f" ✗ {model_id}: {error}", file=sys.stderr)
|
print(f" ✗ {model_id}: {error}", file=sys.stderr)
|
||||||
|
|||||||
@@ -1,34 +1,47 @@
|
|||||||
"""Оба пути пробы запрашивают none и не падают на reasoning-only ответе."""
|
"""Оба пути пробы принимают текст в content или reasoning_content."""
|
||||||
import json
|
|
||||||
import sqlite3
|
import sqlite3
|
||||||
|
|
||||||
|
|
||||||
def test_cron_disables_reasoning_and_reports_null_content(core, omni_server):
|
def test_cron_accepts_reasoning_only(core, omni_server):
|
||||||
omni_server.set_completions(content=None, reasoning_content=" thinking ", finish_reason="length",
|
omni_server.set_completions(content=None, reasoning_content=" thinking ", finish_reason="length",
|
||||||
usage={"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17})
|
usage={"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17})
|
||||||
ok, latency, error, usage = core.probe.check_model("zai/test-model-a")
|
ok, latency, error, usage = core.probe.check_model("zai/test-model-a")
|
||||||
assert ok is False
|
assert ok is True and error is None
|
||||||
assert error == "Empty text response (finish_reason=length)"
|
|
||||||
assert usage["total_tokens"] == 17
|
assert usage["total_tokens"] == 17
|
||||||
request = omni_server.requests[-1]["body"]
|
request = omni_server.requests[-1]["body"]
|
||||||
assert request["reasoning_effort"] == "none"
|
assert request["reasoning_effort"] == "none"
|
||||||
assert request["max_tokens"] == 5
|
assert request["max_tokens"] == 5
|
||||||
|
|
||||||
|
|
||||||
def test_manual_disables_reasoning_and_persists_empty_text(core, omni_server):
|
def test_manual_accepts_reasoning_only_and_persists_usage(core, omni_server):
|
||||||
omni_server.set_completions(content=None, reasoning_content="thinking", finish_reason="length",
|
omni_server.set_completions(content=None, reasoning_content="thinking", finish_reason="length",
|
||||||
usage={"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17})
|
usage={"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17})
|
||||||
response = core.client.get("/api/check/zai/test-model-a")
|
response = core.client.get("/api/check/zai/test-model-a")
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
assert response.json()["ok"] is False
|
assert response.json()["ok"] is True
|
||||||
assert omni_server.requests[-1]["body"]["reasoning_effort"] == "none"
|
assert omni_server.requests[-1]["body"]["reasoning_effort"] == "none"
|
||||||
db = sqlite3.connect(core.db_path)
|
db = sqlite3.connect(core.db_path)
|
||||||
row = db.execute("SELECT ok,error,total_tokens FROM checks ORDER BY id DESC LIMIT 1").fetchone()
|
row = db.execute("SELECT ok,error,total_tokens FROM checks ORDER BY id DESC LIMIT 1").fetchone()
|
||||||
db.close()
|
db.close()
|
||||||
assert row == (0, "Empty text response (finish_reason=length)", 17)
|
assert row == (1, None, 17)
|
||||||
|
|
||||||
|
|
||||||
|
def test_both_fields_empty_are_not_success(core, omni_server):
|
||||||
|
omni_server.set_completions(content=" ", reasoning_content="\t", finish_reason="length",
|
||||||
|
usage={"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17})
|
||||||
|
ok, _, error, usage = core.probe.check_model("zai/test-model-a")
|
||||||
|
assert ok is False
|
||||||
|
assert error == "Empty content and reasoning (finish_reason=length)"
|
||||||
|
assert usage["total_tokens"] == 17
|
||||||
|
response = core.client.get("/api/check/zai/test-model-a")
|
||||||
|
assert response.json()["ok"] is False
|
||||||
|
db = sqlite3.connect(core.db_path)
|
||||||
|
row = db.execute("SELECT ok,error,total_tokens FROM checks ORDER BY id DESC LIMIT 1").fetchone()
|
||||||
|
db.close()
|
||||||
|
assert row == (0, "Empty content and reasoning (finish_reason=length)", 17)
|
||||||
|
|
||||||
|
|
||||||
def test_text_response_still_succeeds(core, omni_server):
|
def test_text_response_still_succeeds(core, omni_server):
|
||||||
omni_server.set_completions(content=" 4 ")
|
omni_server.set_completions(content=" 4 ", reasoning_content=None)
|
||||||
assert core.probe.check_model("zai/test-model-a")[0] is True
|
assert core.probe.check_model("zai/test-model-a")[0] is True
|
||||||
assert core.client.get("/api/check/zai/test-model-a").json()["ok"] is True
|
assert core.client.get("/api/check/zai/test-model-a").json()["ok"] is True
|
||||||
|
|||||||
Reference in New Issue
Block a user