Files
sales-trainer/backend/tests/test_llm_failure_logging.py
Macky 9df2938557 fix(llm): log raw provider output on JSON failure, raise judge budget to 6000
- complete_json(): log model/max_tokens + raw provider text (1500 cap) when
  json.loads fails, so a truncated judge reply is diagnosable from the log
- complete()/complete_conversation(): log reasoning_content head on empty
  content (hidden reasoning starves the shared max_tokens cap)
- simulator.judge(): max_tokens 2000 -> 6000 (server accepts 8000)
- tests: pin judge budget; new test_llm_failure_logging.py pins the logging
  (autouse fixture restores app.llm logger disabled by alembic fileConfig)

547 passed. Incident #4 (500 char 125).
2026-10-02 22:30:28 +07:00

84 lines
2.7 KiB
Python

"""Failure-path logging: when the LLM returns junk (truncated JSON, empty
content), the server log must show WHAT the LLM sent — the raw output and/or
the hidden reasoning head — so live 500s are debuggable from the log alone
(2026-10-02 incident: 'Unterminated string char 125' with zero raw output in
the log).
Bypass __init__ (no network / no OpenAI client) and stub _create.
"""
from __future__ import annotations
import logging
import pytest
from app.llm import LLMClient, LLMError
@pytest.fixture(autouse=True)
def _logging_capture_restored():
# alembic's env.py (run by test_db_schema) does fileConfig(alembic.ini) with
# disable_existing_loggers=True — that disables `app.llm`'s logger (and the
# root) for the rest of the session, so caplog captures nothing. Restore.
def _restore():
logging.getLogger("app.llm").disabled = False
logging.getLogger().disabled = False
_restore()
yield
_restore()
def _client_with_content(content: str, reasoning: str = "") -> LLMClient:
client = LLMClient.__new__(LLMClient)
client.model = "test-model"
class _Msg:
pass
class _Choice:
message = _Msg()
class _Resp:
choices = [_Choice()]
resp = _Resp()
resp.choices[0].message.content = content
resp.choices[0].message.reasoning_content = reasoning
client._create = lambda **kw: resp
return client
def test_invalid_json_logs_raw_output(caplog):
raw = '{\n "outcome": "lo' # truncated mid-string, as in the live incident
client = _client_with_content(raw)
with caplog.at_level("ERROR"):
with pytest.raises(LLMError) as exc:
client.complete_json("sys", "user")
assert "invalid JSON" in str(exc.value)
# The raw output must be visible in the log, capped and delimited.
assert "raw output" in caplog.text
assert '"outcome": "lo' in caplog.text
def test_empty_content_logs_reasoning_head(caplog):
client = _client_with_content(
"", reasoning="hidden chain-of-thought that consumed the budget"
)
with caplog.at_level("ERROR"):
with pytest.raises(LLMError):
client.complete("sys", "user", max_tokens=2000)
assert "empty content" in caplog.text
assert "hidden chain-of-thought that consumed the budget" in caplog.text
def test_empty_conversation_logs_reasoning_head(caplog):
client = _client_with_content(
"", reasoning="reasoning that ate the whole cap"
)
with caplog.at_level("ERROR"):
with pytest.raises(LLMError):
client.complete_conversation([{"role": "seller", "text": "hi"}])
assert "empty content" in caplog.text
assert "reasoning that ate the whole cap" in caplog.text