- complete_json(): log model/max_tokens + raw provider text (1500 cap) when json.loads fails, so a truncated judge reply is diagnosable from the log - complete()/complete_conversation(): log reasoning_content head on empty content (hidden reasoning starves the shared max_tokens cap) - simulator.judge(): max_tokens 2000 -> 6000 (server accepts 8000) - tests: pin judge budget; new test_llm_failure_logging.py pins the logging (autouse fixture restores app.llm logger disabled by alembic fileConfig) 547 passed. Incident #4 (500 char 125).
84 lines
2.7 KiB
Python
84 lines
2.7 KiB
Python
"""Failure-path logging: when the LLM returns junk (truncated JSON, empty
|
|
content), the server log must show WHAT the LLM sent — the raw output and/or
|
|
the hidden reasoning head — so live 500s are debuggable from the log alone
|
|
(2026-10-02 incident: 'Unterminated string char 125' with zero raw output in
|
|
the log).
|
|
|
|
Bypass __init__ (no network / no OpenAI client) and stub _create.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
|
|
import pytest
|
|
|
|
from app.llm import LLMClient, LLMError
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _logging_capture_restored():
|
|
# alembic's env.py (run by test_db_schema) does fileConfig(alembic.ini) with
|
|
# disable_existing_loggers=True — that disables `app.llm`'s logger (and the
|
|
# root) for the rest of the session, so caplog captures nothing. Restore.
|
|
def _restore():
|
|
logging.getLogger("app.llm").disabled = False
|
|
logging.getLogger().disabled = False
|
|
|
|
_restore()
|
|
yield
|
|
_restore()
|
|
|
|
|
|
def _client_with_content(content: str, reasoning: str = "") -> LLMClient:
|
|
client = LLMClient.__new__(LLMClient)
|
|
client.model = "test-model"
|
|
|
|
class _Msg:
|
|
pass
|
|
|
|
class _Choice:
|
|
message = _Msg()
|
|
|
|
class _Resp:
|
|
choices = [_Choice()]
|
|
|
|
resp = _Resp()
|
|
resp.choices[0].message.content = content
|
|
resp.choices[0].message.reasoning_content = reasoning
|
|
client._create = lambda **kw: resp
|
|
return client
|
|
|
|
|
|
def test_invalid_json_logs_raw_output(caplog):
|
|
raw = '{\n "outcome": "lo' # truncated mid-string, as in the live incident
|
|
client = _client_with_content(raw)
|
|
with caplog.at_level("ERROR"):
|
|
with pytest.raises(LLMError) as exc:
|
|
client.complete_json("sys", "user")
|
|
assert "invalid JSON" in str(exc.value)
|
|
# The raw output must be visible in the log, capped and delimited.
|
|
assert "raw output" in caplog.text
|
|
assert '"outcome": "lo' in caplog.text
|
|
|
|
|
|
def test_empty_content_logs_reasoning_head(caplog):
|
|
client = _client_with_content(
|
|
"", reasoning="hidden chain-of-thought that consumed the budget"
|
|
)
|
|
with caplog.at_level("ERROR"):
|
|
with pytest.raises(LLMError):
|
|
client.complete("sys", "user", max_tokens=2000)
|
|
assert "empty content" in caplog.text
|
|
assert "hidden chain-of-thought that consumed the budget" in caplog.text
|
|
|
|
|
|
def test_empty_conversation_logs_reasoning_head(caplog):
|
|
client = _client_with_content(
|
|
"", reasoning="reasoning that ate the whole cap"
|
|
)
|
|
with caplog.at_level("ERROR"):
|
|
with pytest.raises(LLMError):
|
|
client.complete_conversation([{"role": "seller", "text": "hi"}])
|
|
assert "empty content" in caplog.text
|
|
assert "reasoning that ate the whole cap" in caplog.text
|