Files

89 lines
3.9 KiB
Python

"""Budget guard: the Qwen GGUF reasons internally (hidden thinking, kept in a
separate field via enable_thinking=true) before the visible answer. If a budget is
too small, the reply has no headroom and the call can return content='' → LLMError
→ "LLM service unavailable" (2026-10-02 live incident: persona_reply at 400 tokens,
judge JSON at 800).
These tests pin the budgets so an "optimization" back to the small values fails loudly
instead of silently re-breaking chat.
"""
from __future__ import annotations
from app.services.simulator import Simulator
class _CapturingLLM:
def __init__(self, payload: str) -> None:
self.payload = payload
self.conversation_kwargs: list[dict] = []
self.json_kwargs: list[dict] = []
def complete_conversation(self, messages, **kwargs):
self.conversation_kwargs.append(kwargs)
return self.payload
def complete_json(self, system, user, **kwargs):
self.json_kwargs.append(kwargs)
return {"decision": "none", "mood": 0, "reason": "x", "summary": "s"}
def test_persona_reply_budget_covers_hidden_reasoning():
# 1200: enough for the model's internal reasoning + the visible reply.
stub = _CapturingLLM('{"reply": "ค่ะ"}')
Simulator(stub).persona_reply(
persona={
"name": "n", "tier": "B", "initiation_mode": "customer", "channel": "line",
"profession": "p", "age_group": "u", "background": "b", "personality": "r",
"lifestyle": "l", "income": "i", "budget": "b", "decision_timeline": "d",
"pains": [], "negotiation_levers": [], "goal": "g", "tolerance": 2,
"recontact": True, "special": "",
},
sales_kit={"productName": "x"},
messages=[{"role": "seller", "text": "สวัสดีครับ"}],
internal={"turns": 1, "misses": 0, "score": 50},
scenario="social",
)
assert stub.conversation_kwargs[0]["max_tokens"] >= 1200
def test_judge_budget_covers_hidden_reasoning():
# 1600: the judge JSON (score/mood/decision/reason/summary) must fit after
# the hidden reasoning, or the judge silently degrades to 'pending' and a
# customer who decides to buy never closes the session.
stub = _CapturingLLM("")
Simulator(stub).evaluate_turn(
persona={"name": "n", "tier": "B", "budget": "b"},
messages=[{"role": "customer", "text": "hi"}, {"role": "seller", "text": "hello"}],
internal={"turns": 1, "misses": 0, "score": 50},
)
assert stub.json_kwargs[0]["max_tokens"] >= 1600
def test_public_debrief_budget_covers_hidden_reasoning():
# The public coaching pass (why / failurePoints / coaching) analyzes the full
# seller transcript. Same hidden-reasoning truncation class as the final judge
# (500 #4): at 1200 the internal chain ate the budget, the JSON got truncated,
# complete_json threw LLMError, and _public_debrief failed closed to {} — so the
# loss debrief showed only the outcome banner + persona, no "why" and no coaching.
# 6000 matches the final judge.
stub = _CapturingLLM("")
Simulator(stub).public_debrief(
messages=[{"role": "seller", "text": "สวัสดี"}],
outcome="lost",
score=30,
)
assert stub.json_kwargs[0]["max_tokens"] >= 6000
def test_final_judge_budget_covers_hidden_reasoning():
# The FINAL judge (end-of-session verdict) is the fatal path — a truncation
# there 500s the whole chat send (the 2026-10-02 live incident: JSON cut off
# mid-string at char ~125 because the hidden reasoning consumed the 2000 cap).
# 6000 leaves headroom for a long internal chain + the 7-field JSON.
stub = _CapturingLLM("")
Simulator(stub).judge(
persona={"name": "n", "tier": "B", "budget": "b"},
messages=[{"role": "seller", "text": "สวัสดี"}, {"role": "customer", "text": "ตกลง"}],
)
assert stub.json_kwargs[0]["max_tokens"] >= 6000