refactor(chat): decide buy/walk by per-turn LLM judge (not fixed keywords)
Removed the fixed-value text detector. persona_reply no longer forces JSON meta; instead a per-turn evaluate_turn() calls the judge LLM after every customer reply to read the persona's current mood + whether it has decided (buy/walk/pending) + score_delta + reason. send_message consumes that context-based decision to (a) end the chat as won/lost and (b) move the score. This is what the user asked: the system evaluates EVERY turn and decides at the moment it's truly committed — not keyword matching (so 'ซื้อไม่ไหว แต่ว่ามีผ่อนไหม?' stays pending). Mock updated: judge returns buy on first send (keeps E2E deterministic). 11/11 suites pass.
This commit is contained in:
@@ -143,19 +143,63 @@ class Simulator:
|
||||
resp = self.llm.complete_conversation(msgs, temperature=0.7, max_tokens=400)
|
||||
except LLMError as exc:
|
||||
raise
|
||||
# extract {reply, decision, mood}
|
||||
meta: dict[str, Any] = {"decision": "none", "mood": 0}
|
||||
# Return the customer's reply as plain text (no forced JSON) — a natural sentence is
|
||||
# the persona's message. Mood/decision are evaluated separately by evaluate_turn().
|
||||
reply = resp.strip() if resp and resp.strip() else "(ลูกค้ายังไม่ตอบ)"
|
||||
return reply, {"decision": "none", "mood": 0}
|
||||
|
||||
def evaluate_turn(self, *, persona, messages, internal=None) -> dict[str, Any]:
|
||||
"""Per-turn state evaluation: how the persona feels + whether it has decided.
|
||||
|
||||
Uses the SAME judge LLM (structured JSON) on every round so the win/loss decision is
|
||||
derived from the conversation context — NOT from fixed keywords. Returns
|
||||
{mood, decision(buy|walk|pending), score_delta, reason}.
|
||||
"""
|
||||
transcript = "\n".join(
|
||||
f"{m.get('role')}: {m.get('text')}" for m in messages[-30:]
|
||||
)
|
||||
persona_summary = json.dumps({
|
||||
"name": persona.get("name", "?"),
|
||||
"pains": persona.get("pains", []),
|
||||
"budget": persona.get("budget", ""),
|
||||
"tolerance": persona.get("tolerance", 3),
|
||||
"negotiation_levers": persona.get("negotiation_levers", []),
|
||||
"special": persona.get("special", ""),
|
||||
"recontact": persona.get("recontact", False),
|
||||
"goal": persona.get("goal", ""),
|
||||
"decision_timeline": persona.get("decision_timeline", ""),
|
||||
}, ensure_ascii=False)
|
||||
state_note = ""
|
||||
if internal:
|
||||
state_note = (
|
||||
f"\n\nINTERNAL (hidden, judging only): turns={internal.get('turns', 0)}, "
|
||||
f"misses={internal.get('misses', 0)}, score={internal.get('score', 50)}"
|
||||
)
|
||||
sys = (
|
||||
"You are a neutral sales-coaching judge. Read the TRANSCRIPT and decide, as the "
|
||||
"customer persona, how it CURRENTLY feels and whether it has made a decision.\n"
|
||||
"- mood: -2..+2 (very negative .. very positive toward purchase)\n"
|
||||
"- decision: 'buy' if the customer has clearly decided to buy, 'walk' if clearly "
|
||||
"refused/walking away (cannot afford / no interest), else 'pending' (still deciding)\n"
|
||||
"- score_delta: -15..+15 (direction of the sale after this turn)\n"
|
||||
"- reason: one short Thai/English sentence matching the transcript language.\n"
|
||||
"Only output valid JSON: {mood, decision, score_delta, reason}."
|
||||
)
|
||||
user_prompt = f"PERSONA:\n{persona_summary}\n\nTRANSCRIPT:\n{transcript}{state_note}"
|
||||
try:
|
||||
data = json.loads(self._extract_json(resp))
|
||||
reply = (data.get("reply") or data.get("response") or str(resp)).strip()
|
||||
meta["decision"] = data.get("decision", "none")
|
||||
try:
|
||||
meta["mood"] = int(float(data.get("mood", 0)))
|
||||
except (TypeError, ValueError):
|
||||
meta["mood"] = 0
|
||||
except Exception:
|
||||
reply = resp.strip()
|
||||
return reply, meta
|
||||
result = self.judge_llm.complete_json(
|
||||
sys, user_prompt, temperature=0.2, max_tokens=800
|
||||
)
|
||||
except LLMError:
|
||||
# judge unavailable — fall back to pending (don't crash, don't misuse keywords)
|
||||
return {"mood": 0, "decision": "pending", "score_delta": 0, "reason": ""}
|
||||
result.setdefault("mood", 0)
|
||||
result.setdefault("decision", "pending")
|
||||
result.setdefault("score_delta", 0)
|
||||
result.setdefault("reason", "")
|
||||
if result.get("decision") not in ("buy", "walk", "pending"):
|
||||
result["decision"] = "pending"
|
||||
return result
|
||||
|
||||
# ── judge ──────────────────────────────────────────────────────────
|
||||
def judge(
|
||||
|
||||
Reference in New Issue
Block a user