diff --git a/app/main.py b/app/main.py index 02a7026..f85caeb 100644 --- a/app/main.py +++ b/app/main.py @@ -654,10 +654,11 @@ def api_check(model_id: str): # упадёт, реально потраченные токены сохраняются в checks. usage = extract_usage(resp) choice = resp["choices"][0] - content = choice["message"].get("content") - ok = isinstance(content, str) and bool(content.strip()) + message = choice["message"] + ok = any(isinstance(message.get(field), str) and message[field].strip() + for field in ("content", "reasoning_content")) if not ok: - error = f"Empty text response (finish_reason={choice.get('finish_reason')})" + error = f"Empty content and reasoning (finish_reason={choice.get('finish_reason')})" except Exception as e: error = f"{type(e).__name__}: {e}" print(f"[check] {model_id}: {error}") diff --git a/app/probe.py b/app/probe.py index b322be8..dac03c3 100644 --- a/app/probe.py +++ b/app/probe.py @@ -51,13 +51,16 @@ def check_model(model_id: str) -> tuple[bool, int | None, str | None, dict]: resp = json.loads(r.read()) # Usage — до любой валидации контента. usage = extract_usage(resp) - # HTTP 200 может содержать только reasoning и content=null. - # Это не успешный текстовый ответ и не ошибка Python. + # HTTP 200 может содержать только reasoning при исчерпании лимита. + # Наличие текста в любом из двух полей достаточно для liveness. choice = resp["choices"][0] - content = choice["message"].get("content") - ok = isinstance(content, str) and bool(content.strip()) + message = choice["message"] + # Liveness: непустой текст в content ИЛИ reasoning_content. + # Usage само по себе не доказывает, что модель выдала ответ. + ok = any(isinstance(message.get(field), str) and message[field].strip() + for field in ("content", "reasoning_content")) if not ok: - error = f"Empty text response (finish_reason={choice.get('finish_reason')})" + error = f"Empty content and reasoning (finish_reason={choice.get('finish_reason')})" except Exception as e: error = f"{type(e).__name__}: {e}" print(f" ✗ {model_id}: {error}", file=sys.stderr) diff --git a/tests/test_reasoning_probe.py b/tests/test_reasoning_probe.py index c024950..bd913f9 100644 --- a/tests/test_reasoning_probe.py +++ b/tests/test_reasoning_probe.py @@ -1,34 +1,47 @@ -"""Оба пути пробы запрашивают none и не падают на reasoning-only ответе.""" -import json +"""Оба пути пробы принимают текст в content или reasoning_content.""" import sqlite3 -def test_cron_disables_reasoning_and_reports_null_content(core, omni_server): - omni_server.set_completions(content=None, reasoning_content="thinking", finish_reason="length", +def test_cron_accepts_reasoning_only(core, omni_server): + omni_server.set_completions(content=None, reasoning_content=" thinking ", finish_reason="length", usage={"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17}) ok, latency, error, usage = core.probe.check_model("zai/test-model-a") - assert ok is False - assert error == "Empty text response (finish_reason=length)" + assert ok is True and error is None assert usage["total_tokens"] == 17 request = omni_server.requests[-1]["body"] assert request["reasoning_effort"] == "none" assert request["max_tokens"] == 5 -def test_manual_disables_reasoning_and_persists_empty_text(core, omni_server): +def test_manual_accepts_reasoning_only_and_persists_usage(core, omni_server): omni_server.set_completions(content=None, reasoning_content="thinking", finish_reason="length", usage={"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17}) response = core.client.get("/api/check/zai/test-model-a") assert response.status_code == 200 - assert response.json()["ok"] is False + assert response.json()["ok"] is True assert omni_server.requests[-1]["body"]["reasoning_effort"] == "none" db = sqlite3.connect(core.db_path) row = db.execute("SELECT ok,error,total_tokens FROM checks ORDER BY id DESC LIMIT 1").fetchone() db.close() - assert row == (0, "Empty text response (finish_reason=length)", 17) + assert row == (1, None, 17) + + +def test_both_fields_empty_are_not_success(core, omni_server): + omni_server.set_completions(content=" ", reasoning_content="\t", finish_reason="length", + usage={"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17}) + ok, _, error, usage = core.probe.check_model("zai/test-model-a") + assert ok is False + assert error == "Empty content and reasoning (finish_reason=length)" + assert usage["total_tokens"] == 17 + response = core.client.get("/api/check/zai/test-model-a") + assert response.json()["ok"] is False + db = sqlite3.connect(core.db_path) + row = db.execute("SELECT ok,error,total_tokens FROM checks ORDER BY id DESC LIMIT 1").fetchone() + db.close() + assert row == (0, "Empty content and reasoning (finish_reason=length)", 17) def test_text_response_still_succeeds(core, omni_server): - omni_server.set_completions(content=" 4 ") + omni_server.set_completions(content=" 4 ", reasoning_content=None) assert core.probe.check_model("zai/test-model-a")[0] is True assert core.client.get("/api/check/zai/test-model-a").json()["ok"] is True