89 lines
3.9 KiB
Python
89 lines
3.9 KiB
Python
"""Budget guard: the Qwen GGUF reasons internally (hidden thinking, kept in a
|
|
separate field via enable_thinking=true) before the visible answer. If a budget is
|
|
too small, the reply has no headroom and the call can return content='' → LLMError
|
|
→ "LLM service unavailable" (2026-10-02 live incident: persona_reply at 400 tokens,
|
|
judge JSON at 800).
|
|
|
|
These tests pin the budgets so an "optimization" back to the small values fails loudly
|
|
instead of silently re-breaking chat.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from app.services.simulator import Simulator
|
|
|
|
|
|
class _CapturingLLM:
|
|
def __init__(self, payload: str) -> None:
|
|
self.payload = payload
|
|
self.conversation_kwargs: list[dict] = []
|
|
self.json_kwargs: list[dict] = []
|
|
|
|
def complete_conversation(self, messages, **kwargs):
|
|
self.conversation_kwargs.append(kwargs)
|
|
return self.payload
|
|
|
|
def complete_json(self, system, user, **kwargs):
|
|
self.json_kwargs.append(kwargs)
|
|
return {"decision": "none", "mood": 0, "reason": "x", "summary": "s"}
|
|
|
|
|
|
def test_persona_reply_budget_covers_hidden_reasoning():
|
|
# 1200: enough for the model's internal reasoning + the visible reply.
|
|
stub = _CapturingLLM('{"reply": "ค่ะ"}')
|
|
Simulator(stub).persona_reply(
|
|
persona={
|
|
"name": "n", "tier": "B", "initiation_mode": "customer", "channel": "line",
|
|
"profession": "p", "age_group": "u", "background": "b", "personality": "r",
|
|
"lifestyle": "l", "income": "i", "budget": "b", "decision_timeline": "d",
|
|
"pains": [], "negotiation_levers": [], "goal": "g", "tolerance": 2,
|
|
"recontact": True, "special": "",
|
|
},
|
|
sales_kit={"productName": "x"},
|
|
messages=[{"role": "seller", "text": "สวัสดีครับ"}],
|
|
internal={"turns": 1, "misses": 0, "score": 50},
|
|
scenario="social",
|
|
)
|
|
assert stub.conversation_kwargs[0]["max_tokens"] >= 1200
|
|
|
|
|
|
def test_judge_budget_covers_hidden_reasoning():
|
|
# 1600: the judge JSON (score/mood/decision/reason/summary) must fit after
|
|
# the hidden reasoning, or the judge silently degrades to 'pending' and a
|
|
# customer who decides to buy never closes the session.
|
|
stub = _CapturingLLM("")
|
|
Simulator(stub).evaluate_turn(
|
|
persona={"name": "n", "tier": "B", "budget": "b"},
|
|
messages=[{"role": "customer", "text": "hi"}, {"role": "seller", "text": "hello"}],
|
|
internal={"turns": 1, "misses": 0, "score": 50},
|
|
)
|
|
assert stub.json_kwargs[0]["max_tokens"] >= 1600
|
|
|
|
|
|
def test_public_debrief_budget_covers_hidden_reasoning():
|
|
# The public coaching pass (why / failurePoints / coaching) analyzes the full
|
|
# seller transcript. Same hidden-reasoning truncation class as the final judge
|
|
# (500 #4): at 1200 the internal chain ate the budget, the JSON got truncated,
|
|
# complete_json threw LLMError, and _public_debrief failed closed to {} — so the
|
|
# loss debrief showed only the outcome banner + persona, no "why" and no coaching.
|
|
# 6000 matches the final judge.
|
|
stub = _CapturingLLM("")
|
|
Simulator(stub).public_debrief(
|
|
messages=[{"role": "seller", "text": "สวัสดี"}],
|
|
outcome="lost",
|
|
score=30,
|
|
)
|
|
assert stub.json_kwargs[0]["max_tokens"] >= 6000
|
|
|
|
|
|
def test_final_judge_budget_covers_hidden_reasoning():
|
|
# The FINAL judge (end-of-session verdict) is the fatal path — a truncation
|
|
# there 500s the whole chat send (the 2026-10-02 live incident: JSON cut off
|
|
# mid-string at char ~125 because the hidden reasoning consumed the 2000 cap).
|
|
# 6000 leaves headroom for a long internal chain + the 7-field JSON.
|
|
stub = _CapturingLLM("")
|
|
Simulator(stub).judge(
|
|
persona={"name": "n", "tier": "B", "budget": "b"},
|
|
messages=[{"role": "seller", "text": "สวัสดี"}, {"role": "customer", "text": "ตกลง"}],
|
|
)
|
|
assert stub.json_kwargs[0]["max_tokens"] >= 6000
|