[verified] fix live-QA UX: no JSON leak in chat, greeting openers (wrong_text cools off post-first-reply), auto-close+summarize on buy/walk/try, remove Finish button

2026-08-18 09:42:05 +07:00
parent 2c40fc7502
commit ffb7cdda31
11 changed files with 379 additions and 46 deletions
--- a/backend/app/api/chat_routes.py
+++ b/backend/app/api/chat_routes.py
@@ -519,7 +519,20 @@ def send_message(gid: str, pid: str):
    elif mood >= 1:
        internal["signals"].append({"turn": internal["turns"], "mood": mood, "type": "warm"})

-    if decision in ("buy", "walk"):
+    if decision in ("buy", "walk", "try"):
+        if decision == "try":
+            # The customer decided to trial/test the product first and will come
+            # back later. Surface that as a system note, then close + summarize
+            # (the decision point is reached; the final judge scores the close).
+            try_note = (
+                "⏳ ลูกค้าตัดสินใจจะลองใช้สินค้า/บริการก่อน แล้วจะกลับมาติดต่ออีกครั้งเมื่อลองใช้แล้ว — "
+                "ถือเป็นการสรุปการตัดสินใจจุดนี้ และปิด session อัตโนมัติ"
+                if slocale != "en"
+                else "⏳ The customer decided to trial the product/service first and will come back "
+                "once they have tried it — this decision point is closed and the session summarizes "
+                "automatically."
+            )
+            messages.append({"role": "system", "text": try_note})
        finalized, debrief = _finalize_session(
            s,
            {**session, "messages": messages, "internal": internal},
--- a/backend/app/services/persona_prompts.py
+++ b/backend/app/services/persona_prompts.py
@@ -43,9 +43,18 @@ RULES:
   OR to walk away (after too many misses / rude / pushy / wrong), the persona STATES the decision in
   ordinary dialogue (e.g. "ok I'll go with it" / "no thanks, forget it") — it does NOT announce it as meta.
 5. CHANNEL: "facebook" or "line".
-6. ONE SPECIAL TIER-C PERSONA: special="wrong_text". They open looking ready to buy, then instantly
-   lose interest and want to end the chat (open='never mind, forget it'), yet still have a live pain.
+6. ONE SPECIAL TIER-C PERSONA: special="wrong_text". They message the seller normally first (see the
+   opener rule below) as if genuinely interested, then AFTER the seller's very first reply they cool off
+   and try to end the chat (e.g. "sorry, wrong chat" / "never mind, forget it"), yet still have a live pain.
+   A seller who gently re-engages without pushing may earn a second chance; a pushy seller drives them away.
 7. difficulty 1-5. special="" unless wrong_text.
-8. Language: output all human text in the requested language.
+8. OPENER RULE (IMPORTANT): the `opener` is what the customer says FIRST when initiation_mode="customer".
+   Every opener — for EVERY tier, including wrong_text and every tier-C persona — must be a natural, polite,
+   in-character customer greeting that starts warmly or neutrally and expresses some interest or a question
+   (e.g. "สวัสดีค่ะ เห็นสินค้าคุณแล้ว สนใจอยากสอบถาม" / "สวัสดีครับ เข้าไปดูเพจมา อยากถามราคาหน่อย" / "Hi, I saw
+   your page and had a question"). NEVER make the opener a complaint, a refusal, a price grumble, a "never
+   mind", or anything negative — the customer's resistance must only surface DURING the conversation, not
+   in their very first message.
+9. Language: output all human text in the requested language.
 Only output valid JSON: {"personas": [ ... ]}
 """
--- a/backend/app/services/simulator.py
+++ b/backend/app/services/simulator.py
@@ -7,6 +7,7 @@ never exposed mid-chat. Initiation is per-persona (customer or seller).
 from __future__ import annotations

 import json
+import re
 from typing import Any

 from ..llm import LLMClient, LLMError
@@ -189,8 +190,10 @@ class Simulator:
            "You are a neutral sales-coaching judge. Read the TRANSCRIPT and decide, as the "
            "customer persona, how it CURRENTLY feels and whether it has made a decision.\n"
            "- mood: -2..+2 (very negative .. very positive toward purchase)\n"
-            "- decision: 'buy' if the customer has clearly decided to buy, 'walk' if clearly "
-            "refused/walking away (cannot afford / no interest), else 'pending' (still deciding)\n"
+            "- decision: 'buy' if the customer has clearly decided to buy now, 'walk' if clearly "
+            "refused/walking away (cannot afford / no interest), 'try' if the customer has decided to "
+            "trial/test the product or service first and will come back later (not a straight refusal, "
+            "not a confirmed purchase), else 'pending' (still deciding)\n"
            "- score_delta: -15..+15 (direction of the sale after this turn)\n"
            "- reason: one short Thai/English sentence matching the transcript language.\n"
            "Only output valid JSON: {mood, decision, score_delta, reason}."
@@ -209,7 +212,7 @@ class Simulator:
        result.setdefault("decision", "pending")
        result.setdefault("score_delta", 0)
        result.setdefault("reason", "")
-        if result.get("decision") not in ("buy", "walk", "pending"):
+        if result.get("decision") not in ("buy", "walk", "try", "pending"):
            result["decision"] = "pending"
        return result

@@ -292,7 +295,9 @@ class Simulator:
        The preferred contract is a JSON object. A plain natural-language
        response remains a safe compatibility fallback for providers that ignore
        JSON mode. Objects without a usable ``reply`` are retryable; they are
-        never rendered verbatim.
+        never rendered verbatim. A malformed/partial protocol blob (even one
+        missing its opening brace) is NEVER surfaced — it is rejected and
+        triggers a bounded retry instead of leaking raw JSON into the chat.
        """
        if not isinstance(text, str):
            return None
@@ -328,18 +333,72 @@ class Simulator:
                "mood": mood,
            }

-        # A plain sentence is safe; protocol-looking malformed JSON is not.
-        if raw.startswith(("{", "[", "```")):
+        # Reject anything that still looks like a protocol envelope or partial
+        # JSON (e.g. a fenced block, a leading brace/bracket, or text carrying
+        # the contract's reserved keys). These must never render in the bubble.
+        if self._looks_like_protocol(raw, extracted):
            return None
-        return raw[:MAX_PERSONA_REPLY_CHARS], {"decision": "none", "mood": 0}
+        # A leading quote-wrapped reply with trailing key:value noise can reach
+        # here; drop the trailing noise so only the spoken sentence is kept.
+        cleaned = self._strip_trailing_protocol_noise(raw)
+        if not cleaned:
+            return None
+        return cleaned[:MAX_PERSONA_REPLY_CHARS], {"decision": "none", "mood": 0}
+
+    def _looks_like_protocol(self, raw: str, extracted: str) -> bool:
+        """True when the provider text still carries a JSON/protocol signature.
+
+        Covers the exact leakage observed in the wild: a valid-looking reply
+        glued to `"decision": ...` / `"mood": ...` keys with or without braces.
+        """
+        if raw.startswith(("{", "[", "```")):
+            return True
+        # Reserved contract keys mark incomplete protocol output.
+        lowered = raw.lower()
+        if '"reply"' in raw or '"decision"' in lowered or '"mood"' in lowered:
+            return True
+        if "'reply'" in raw or "'decision'" in lowered or "'mood'" in lowered:
+            return True
+        # A colon immediately after a quoted fragment implies key:value noise.
+        if ":" in raw and (extracted.startswith('"') or "{" in raw or "}" in raw):
+            return True
+        return False
+
+    def _strip_trailing_protocol_noise(self, raw: str) -> str:
+        """Remove a trailing `"key": value` fragment from a spoken reply.
+
+        Providers sometimes append contract fields as plain text after the
+        sentence. If they do, keep only the leading natural-language portion.
+        Also unwraps a single leading/trailing quote pair so a bare quoted
+        sentence (`"สวัสดีค่ะ"`) returns clean text instead of quotes.
+        """
+        # Find the first occurrence of a JSON-ish key fragment: `"word"` or `'word'` followed by ':'.
+        match = re.search(r'''["'][A-Za-z_]+["']\s*:''', raw)
+        if match:
+            head = raw[: match.start()].strip().rstrip('"').strip()
+            # Recurse to drop any earlier key-fragments too (rare chained noise).
+            if head and head != raw:
+                sub = self._strip_trailing_protocol_noise(head)
+                return sub or head
+            return head
+        cleaned = raw.strip()
+        # Unwrap a matching leading+trailing quote pair (then retry once).
+        if len(cleaned) >= 2 and cleaned[0] == cleaned[-1] and cleaned[0] in ('"', "'"):
+            inner = cleaned[1:-1].strip()
+            if inner:
+                return inner
+            return cleaned
+        return cleaned.rstrip('"').strip()

    def _special_instr(self, persona: dict[str, Any]) -> str:
        if persona.get("special") == "wrong_text":
            return (
-                "SPECIAL: You opened as if ready to buy, but the moment the seller replies you act "
-                "disinterested and try to end the chat (e.g. 'never mind, forget it'). Deep down your "
-                "pain is still real. A seller who gently re-engages without pushing may earn a second "
-                "chance; a pushy seller drives you away for good."
+                "SPECIAL: You messaged this seller normally (your opener was a genuine greeting/interest "
+                "message — you have not complained and you have NOT said 'wrong chat' or 'never mind' yet). "
+                "ONLY your very first reply to the seller's opening is allowed to cool off: sound "
+                "disinterested and try to end the chat (e.g. 'oh, sorry, I think I messaged the wrong "
+                "person' / 'never mind, forget it'). Deep down your pain is still real. A seller who gently "
+                "re-engages without pushing may earn a second chance; a pushy seller drives you away for good."
            )
        return ""

--- a/backend/tests/test_auto_close_try.py
+++ b/backend/tests/test_auto_close_try.py
@@ -0,0 +1,143 @@
+"""Auto-close contract: when the customer reaches a decision (buy/walk/try),
+the chat route MUST close + summarize automatically — no manual finish needed.
+Covers the 'trial first, come back later' (try) path specifically.
+"""
+from __future__ import annotations
+
+from app.llm import LLMError
+
+
+class StubSim:
+    """Stubbed simulator: a persona reply + a per-turn judge that returns 'try'."""
+
+    def __init__(self, decision: str = "try"):
+        self.decision = decision
+        self.judge_verdict = {
+            "outcome": "lost",
+            "score": 55,
+            "pain": "wants to trial before committing",
+            "why": "customer trialing first, not closed yet",
+            "failurePoints": [],
+            "coaching": [],
+            "painProgress": {"trial": 60},
+        }
+
+    def persona_reply(self, **kwargs):
+        return "ถ้าอย่างนั้น ขอทดลองใช้ก่อนได้ไหมคะ", {"decision": "none", "mood": 1}
+
+    def evaluate_turn(self, **kwargs):
+        return {
+            "mood": 1,
+            "decision": self.decision,
+            "score_delta": 3,
+            "reason": "ลูกค้าตัดสินใจลองใช้ก่อน",
+        }
+
+    def judge(self, **kwargs):
+        return dict(self.judge_verdict)
+
+
+def _auth_headers(token):
+    return {"Authorization": f"Bearer {token}"}
+
+
+def _ready(client, user_store, login):
+    from scripts.mock_llm import MockLLM
+
+    user_store.complete_setup(
+        "admin", "admin@example.com", "admin-ready-password",
+        accepted_terms=True, accepted_terms_at="2026-08-13T00:00:00Z",
+    )
+    token = login("admin", "admin-ready-password")["token"]
+    headers = _auth_headers(token)
+    client.application.extensions["llm"] = MockLLM()
+    response = client.post(
+        "/api/groups", json={"product": "CRM", "segment": "SME"}, headers=headers
+    )
+    assert response.status_code == 201, response.get_json()
+    gid = response.get_json()["group"]["id"]
+    analyzed = client.post(f"/api/groups/{gid}/analyze", headers=headers)
+    assert analyzed.status_code == 200, analyzed.get_json()
+    pid = analyzed.get_json()["personas"][0]["id"]
+    return gid, pid, headers
+
+
+def test_try_decision_auto_closes_with_trial_system_note(client, user_store, login, monkeypatch):
+    gid, pid, headers = _ready(client, user_store, login)
+
+    started = client.post(
+        f"/api/chat/{gid}/personas/{pid}/chat/start",
+        json={"scenario": "social", "locale": "th", "mode": "preview"},
+        headers=headers,
+    )
+    assert started.status_code == 200, started.get_json()
+    assert started.get_json()["session"]["status"] == "active"
+
+    stub = StubSim(decision="try")
+    monkeypatch.setattr("app.api.chat_routes._sim", lambda g, p: stub)
+
+    sent = client.post(
+        f"/api/chat/{gid}/personas/{pid}/chat/send",
+        json={"text": "ได้เลยครับ ลองใช้ก่อนได้"},
+        headers=headers,
+    )
+    assert sent.status_code == 200, sent.get_json()
+    body = sent.get_json()
+
+    # Auto-close: the session must be finished with a summary, no manual step.
+    assert body["finished"] is True
+    assert body["session"]["status"] == "finished"
+    assert body["debrief"] is not None
+    assert body["debrief"]["score"] == 55
+
+    # A system note about the trial + coming back must be in the transcript.
+    texts = [m.get("text", "") for m in body["messages"] if m.get("role") == "system"]
+    assert any("ลองใช้" in t or "trial" in t.lower() for t in texts)
+
+
+def test_buy_decision_auto_closes_won(client, user_store, login, monkeypatch):
+    gid, pid, headers = _ready(client, user_store, login)
+    client.post(
+        f"/api/chat/{gid}/personas/{pid}/chat/start",
+        json={"scenario": "social", "locale": "th", "mode": "preview"},
+        headers=headers,
+    )
+
+    stub = StubSim(decision="buy")
+    stub.judge_verdict["outcome"] = "won"
+    monkeypatch.setattr("app.api.chat_routes._sim", lambda g, p: stub)
+
+    sent = client.post(
+        f"/api/chat/{gid}/personas/{pid}/chat/send",
+        json={"text": "โอเค เอาเลยครับ"},
+        headers=headers,
+    )
+    assert sent.status_code == 200, sent.get_json()
+    body = sent.get_json()
+    assert body["finished"] is True
+    assert body["outcome"] == "won"
+    assert body["session"]["status"] == "finished"
+
+
+def test_walk_decision_auto_closes_lost(client, user_store, login, monkeypatch):
+    gid, pid, headers = _ready(client, user_store, login)
+    client.post(
+        f"/api/chat/{gid}/personas/{pid}/chat/start",
+        json={"scenario": "social", "locale": "th", "mode": "preview"},
+        headers=headers,
+    )
+
+    stub = StubSim(decision="walk")
+    stub.judge_verdict["outcome"] = "lost"
+    monkeypatch.setattr("app.api.chat_routes._sim", lambda g, p: stub)
+
+    sent = client.post(
+        f"/api/chat/{gid}/personas/{pid}/chat/send",
+        json={"text": "งั้นไม่เอาดีกว่าครับ"},
+        headers=headers,
+    )
+    assert sent.status_code == 200, sent.get_json()
+    body = sent.get_json()
+    assert body["finished"] is True
+    assert body["outcome"] == "lost"
+    assert body["session"]["status"] == "finished"
--- a/backend/tests/test_persona_reply_contract.py
+++ b/backend/tests/test_persona_reply_contract.py
@@ -124,3 +124,53 @@ def test_prompt_contains_persona_behavior_contract():
    assert "recontact" in system.lower()
    assert "wrong_text" in system
    assert "tolerance" in system.lower()
+
+
+def test_partial_protocol_without_braces_never_leaks_into_bubble():
+    # Provider appended contract fields as plain text (no braces) — the exact
+    # raw leakage seen in production. Must NOT be rendered verbatim; it must
+    # be rejected so the bounded retry (or fail-closed) path is used.
+    stub = StubLLM([
+        '"สวัสดีค่ะ สนใจสอบถามค่ะ" "decision": "none", "mood": 0',
+        json.dumps({"reply": "สวัสดีค่ะ สนใจสอบถามค่ะ", "decision": "none", "mood": 0}, ensure_ascii=False),
+    ])
+
+    reply, meta = _reply(stub)
+
+    assert reply == "สวัสดีค่ะ สนใจสอบถามค่ะ"
+    assert "decision" not in reply
+    assert "mood" not in reply
+    assert len(stub.calls) == 2
+
+
+def test_malformed_brace_payload_after_retry_fails_closed():
+    stub = StubLLM([
+        '{"reply". "สวัสดีค่ะ" "\'decision": "none", "mood"\' 0}',
+        '{"reply". "ขอลองก่อนนะคะ" "\'decision": "walk"\'}',
+    ])
+
+    with pytest.raises(LLMError, match="persona reply"):
+        _reply(stub)
+
+    assert len(stub.calls) == 2
+
+
+def test_quoted_sentence_unwraps_quotes():
+    stub = StubLLM(['"สวัสดีค่ะ พอดีสนใจสินค้าค่ะ"'])
+
+    reply, _ = _reply(stub)
+
+    assert reply == "สวัสดีค่ะ พอดีสนใจสินค้าค่ะ"
+    assert '"' not in reply
+
+
+def test_natural_thai_with_colon_is_not_stripped_as_protocol_noise():
+    # A legit spoken reply containing a colon (e.g. a time) must NOT be treated
+    # as trailing protocol noise — the heuristic must not over-strip real text.
+    msg = "เจอกันพรุ่งนี้ 09:00 นะครับ"
+    stub = StubLLM([msg])
+
+    reply, _ = _reply(stub)
+
+    assert reply == msg
+    assert "09:00" in reply