[verified] fix live-QA UX: no JSON leak in chat, greeting openers (wrong_text cools off post-first-reply), auto-close+summarize on buy/walk/try, remove Finish button
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -29,3 +29,6 @@ data/
|
||||
# Logs
|
||||
*.log
|
||||
backend/logs/
|
||||
|
||||
# Hermes local agent config (never commit)
|
||||
.hermes/
|
||||
|
||||
@@ -519,7 +519,20 @@ def send_message(gid: str, pid: str):
|
||||
elif mood >= 1:
|
||||
internal["signals"].append({"turn": internal["turns"], "mood": mood, "type": "warm"})
|
||||
|
||||
if decision in ("buy", "walk"):
|
||||
if decision in ("buy", "walk", "try"):
|
||||
if decision == "try":
|
||||
# The customer decided to trial/test the product first and will come
|
||||
# back later. Surface that as a system note, then close + summarize
|
||||
# (the decision point is reached; the final judge scores the close).
|
||||
try_note = (
|
||||
"⏳ ลูกค้าตัดสินใจจะลองใช้สินค้า/บริการก่อน แล้วจะกลับมาติดต่ออีกครั้งเมื่อลองใช้แล้ว — "
|
||||
"ถือเป็นการสรุปการตัดสินใจจุดนี้ และปิด session อัตโนมัติ"
|
||||
if slocale != "en"
|
||||
else "⏳ The customer decided to trial the product/service first and will come back "
|
||||
"once they have tried it — this decision point is closed and the session summarizes "
|
||||
"automatically."
|
||||
)
|
||||
messages.append({"role": "system", "text": try_note})
|
||||
finalized, debrief = _finalize_session(
|
||||
s,
|
||||
{**session, "messages": messages, "internal": internal},
|
||||
|
||||
@@ -43,9 +43,18 @@ RULES:
|
||||
OR to walk away (after too many misses / rude / pushy / wrong), the persona STATES the decision in
|
||||
ordinary dialogue (e.g. "ok I'll go with it" / "no thanks, forget it") — it does NOT announce it as meta.
|
||||
5. CHANNEL: "facebook" or "line".
|
||||
6. ONE SPECIAL TIER-C PERSONA: special="wrong_text". They open looking ready to buy, then instantly
|
||||
lose interest and want to end the chat (open='never mind, forget it'), yet still have a live pain.
|
||||
6. ONE SPECIAL TIER-C PERSONA: special="wrong_text". They message the seller normally first (see the
|
||||
opener rule below) as if genuinely interested, then AFTER the seller's very first reply they cool off
|
||||
and try to end the chat (e.g. "sorry, wrong chat" / "never mind, forget it"), yet still have a live pain.
|
||||
A seller who gently re-engages without pushing may earn a second chance; a pushy seller drives them away.
|
||||
7. difficulty 1-5. special="" unless wrong_text.
|
||||
8. Language: output all human text in the requested language.
|
||||
8. OPENER RULE (IMPORTANT): the `opener` is what the customer says FIRST when initiation_mode="customer".
|
||||
Every opener — for EVERY tier, including wrong_text and every tier-C persona — must be a natural, polite,
|
||||
in-character customer greeting that starts warmly or neutrally and expresses some interest or a question
|
||||
(e.g. "สวัสดีค่ะ เห็นสินค้าคุณแล้ว สนใจอยากสอบถาม" / "สวัสดีครับ เข้าไปดูเพจมา อยากถามราคาหน่อย" / "Hi, I saw
|
||||
your page and had a question"). NEVER make the opener a complaint, a refusal, a price grumble, a "never
|
||||
mind", or anything negative — the customer's resistance must only surface DURING the conversation, not
|
||||
in their very first message.
|
||||
9. Language: output all human text in the requested language.
|
||||
Only output valid JSON: {"personas": [ ... ]}
|
||||
"""
|
||||
|
||||
@@ -7,6 +7,7 @@ never exposed mid-chat. Initiation is per-persona (customer or seller).
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from ..llm import LLMClient, LLMError
|
||||
@@ -189,8 +190,10 @@ class Simulator:
|
||||
"You are a neutral sales-coaching judge. Read the TRANSCRIPT and decide, as the "
|
||||
"customer persona, how it CURRENTLY feels and whether it has made a decision.\n"
|
||||
"- mood: -2..+2 (very negative .. very positive toward purchase)\n"
|
||||
"- decision: 'buy' if the customer has clearly decided to buy, 'walk' if clearly "
|
||||
"refused/walking away (cannot afford / no interest), else 'pending' (still deciding)\n"
|
||||
"- decision: 'buy' if the customer has clearly decided to buy now, 'walk' if clearly "
|
||||
"refused/walking away (cannot afford / no interest), 'try' if the customer has decided to "
|
||||
"trial/test the product or service first and will come back later (not a straight refusal, "
|
||||
"not a confirmed purchase), else 'pending' (still deciding)\n"
|
||||
"- score_delta: -15..+15 (direction of the sale after this turn)\n"
|
||||
"- reason: one short Thai/English sentence matching the transcript language.\n"
|
||||
"Only output valid JSON: {mood, decision, score_delta, reason}."
|
||||
@@ -209,7 +212,7 @@ class Simulator:
|
||||
result.setdefault("decision", "pending")
|
||||
result.setdefault("score_delta", 0)
|
||||
result.setdefault("reason", "")
|
||||
if result.get("decision") not in ("buy", "walk", "pending"):
|
||||
if result.get("decision") not in ("buy", "walk", "try", "pending"):
|
||||
result["decision"] = "pending"
|
||||
return result
|
||||
|
||||
@@ -292,7 +295,9 @@ class Simulator:
|
||||
The preferred contract is a JSON object. A plain natural-language
|
||||
response remains a safe compatibility fallback for providers that ignore
|
||||
JSON mode. Objects without a usable ``reply`` are retryable; they are
|
||||
never rendered verbatim.
|
||||
never rendered verbatim. A malformed/partial protocol blob (even one
|
||||
missing its opening brace) is NEVER surfaced — it is rejected and
|
||||
triggers a bounded retry instead of leaking raw JSON into the chat.
|
||||
"""
|
||||
if not isinstance(text, str):
|
||||
return None
|
||||
@@ -328,18 +333,72 @@ class Simulator:
|
||||
"mood": mood,
|
||||
}
|
||||
|
||||
# A plain sentence is safe; protocol-looking malformed JSON is not.
|
||||
if raw.startswith(("{", "[", "```")):
|
||||
# Reject anything that still looks like a protocol envelope or partial
|
||||
# JSON (e.g. a fenced block, a leading brace/bracket, or text carrying
|
||||
# the contract's reserved keys). These must never render in the bubble.
|
||||
if self._looks_like_protocol(raw, extracted):
|
||||
return None
|
||||
return raw[:MAX_PERSONA_REPLY_CHARS], {"decision": "none", "mood": 0}
|
||||
# A leading quote-wrapped reply with trailing key:value noise can reach
|
||||
# here; drop the trailing noise so only the spoken sentence is kept.
|
||||
cleaned = self._strip_trailing_protocol_noise(raw)
|
||||
if not cleaned:
|
||||
return None
|
||||
return cleaned[:MAX_PERSONA_REPLY_CHARS], {"decision": "none", "mood": 0}
|
||||
|
||||
def _looks_like_protocol(self, raw: str, extracted: str) -> bool:
|
||||
"""True when the provider text still carries a JSON/protocol signature.
|
||||
|
||||
Covers the exact leakage observed in the wild: a valid-looking reply
|
||||
glued to `"decision": ...` / `"mood": ...` keys with or without braces.
|
||||
"""
|
||||
if raw.startswith(("{", "[", "```")):
|
||||
return True
|
||||
# Reserved contract keys mark incomplete protocol output.
|
||||
lowered = raw.lower()
|
||||
if '"reply"' in raw or '"decision"' in lowered or '"mood"' in lowered:
|
||||
return True
|
||||
if "'reply'" in raw or "'decision'" in lowered or "'mood'" in lowered:
|
||||
return True
|
||||
# A colon immediately after a quoted fragment implies key:value noise.
|
||||
if ":" in raw and (extracted.startswith('"') or "{" in raw or "}" in raw):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _strip_trailing_protocol_noise(self, raw: str) -> str:
|
||||
"""Remove a trailing `"key": value` fragment from a spoken reply.
|
||||
|
||||
Providers sometimes append contract fields as plain text after the
|
||||
sentence. If they do, keep only the leading natural-language portion.
|
||||
Also unwraps a single leading/trailing quote pair so a bare quoted
|
||||
sentence (`"สวัสดีค่ะ"`) returns clean text instead of quotes.
|
||||
"""
|
||||
# Find the first occurrence of a JSON-ish key fragment: `"word"` or `'word'` followed by ':'.
|
||||
match = re.search(r'''["'][A-Za-z_]+["']\s*:''', raw)
|
||||
if match:
|
||||
head = raw[: match.start()].strip().rstrip('"').strip()
|
||||
# Recurse to drop any earlier key-fragments too (rare chained noise).
|
||||
if head and head != raw:
|
||||
sub = self._strip_trailing_protocol_noise(head)
|
||||
return sub or head
|
||||
return head
|
||||
cleaned = raw.strip()
|
||||
# Unwrap a matching leading+trailing quote pair (then retry once).
|
||||
if len(cleaned) >= 2 and cleaned[0] == cleaned[-1] and cleaned[0] in ('"', "'"):
|
||||
inner = cleaned[1:-1].strip()
|
||||
if inner:
|
||||
return inner
|
||||
return cleaned
|
||||
return cleaned.rstrip('"').strip()
|
||||
|
||||
def _special_instr(self, persona: dict[str, Any]) -> str:
|
||||
if persona.get("special") == "wrong_text":
|
||||
return (
|
||||
"SPECIAL: You opened as if ready to buy, but the moment the seller replies you act "
|
||||
"disinterested and try to end the chat (e.g. 'never mind, forget it'). Deep down your "
|
||||
"pain is still real. A seller who gently re-engages without pushing may earn a second "
|
||||
"chance; a pushy seller drives you away for good."
|
||||
"SPECIAL: You messaged this seller normally (your opener was a genuine greeting/interest "
|
||||
"message — you have not complained and you have NOT said 'wrong chat' or 'never mind' yet). "
|
||||
"ONLY your very first reply to the seller's opening is allowed to cool off: sound "
|
||||
"disinterested and try to end the chat (e.g. 'oh, sorry, I think I messaged the wrong "
|
||||
"person' / 'never mind, forget it'). Deep down your pain is still real. A seller who gently "
|
||||
"re-engages without pushing may earn a second chance; a pushy seller drives you away for good."
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
143
backend/tests/test_auto_close_try.py
Normal file
143
backend/tests/test_auto_close_try.py
Normal file
@@ -0,0 +1,143 @@
|
||||
"""Auto-close contract: when the customer reaches a decision (buy/walk/try),
|
||||
the chat route MUST close + summarize automatically — no manual finish needed.
|
||||
Covers the 'trial first, come back later' (try) path specifically.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.llm import LLMError
|
||||
|
||||
|
||||
class StubSim:
|
||||
"""Stubbed simulator: a persona reply + a per-turn judge that returns 'try'."""
|
||||
|
||||
def __init__(self, decision: str = "try"):
|
||||
self.decision = decision
|
||||
self.judge_verdict = {
|
||||
"outcome": "lost",
|
||||
"score": 55,
|
||||
"pain": "wants to trial before committing",
|
||||
"why": "customer trialing first, not closed yet",
|
||||
"failurePoints": [],
|
||||
"coaching": [],
|
||||
"painProgress": {"trial": 60},
|
||||
}
|
||||
|
||||
def persona_reply(self, **kwargs):
|
||||
return "ถ้าอย่างนั้น ขอทดลองใช้ก่อนได้ไหมคะ", {"decision": "none", "mood": 1}
|
||||
|
||||
def evaluate_turn(self, **kwargs):
|
||||
return {
|
||||
"mood": 1,
|
||||
"decision": self.decision,
|
||||
"score_delta": 3,
|
||||
"reason": "ลูกค้าตัดสินใจลองใช้ก่อน",
|
||||
}
|
||||
|
||||
def judge(self, **kwargs):
|
||||
return dict(self.judge_verdict)
|
||||
|
||||
|
||||
def _auth_headers(token):
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
def _ready(client, user_store, login):
|
||||
from scripts.mock_llm import MockLLM
|
||||
|
||||
user_store.complete_setup(
|
||||
"admin", "admin@example.com", "admin-ready-password",
|
||||
accepted_terms=True, accepted_terms_at="2026-08-13T00:00:00Z",
|
||||
)
|
||||
token = login("admin", "admin-ready-password")["token"]
|
||||
headers = _auth_headers(token)
|
||||
client.application.extensions["llm"] = MockLLM()
|
||||
response = client.post(
|
||||
"/api/groups", json={"product": "CRM", "segment": "SME"}, headers=headers
|
||||
)
|
||||
assert response.status_code == 201, response.get_json()
|
||||
gid = response.get_json()["group"]["id"]
|
||||
analyzed = client.post(f"/api/groups/{gid}/analyze", headers=headers)
|
||||
assert analyzed.status_code == 200, analyzed.get_json()
|
||||
pid = analyzed.get_json()["personas"][0]["id"]
|
||||
return gid, pid, headers
|
||||
|
||||
|
||||
def test_try_decision_auto_closes_with_trial_system_note(client, user_store, login, monkeypatch):
|
||||
gid, pid, headers = _ready(client, user_store, login)
|
||||
|
||||
started = client.post(
|
||||
f"/api/chat/{gid}/personas/{pid}/chat/start",
|
||||
json={"scenario": "social", "locale": "th", "mode": "preview"},
|
||||
headers=headers,
|
||||
)
|
||||
assert started.status_code == 200, started.get_json()
|
||||
assert started.get_json()["session"]["status"] == "active"
|
||||
|
||||
stub = StubSim(decision="try")
|
||||
monkeypatch.setattr("app.api.chat_routes._sim", lambda g, p: stub)
|
||||
|
||||
sent = client.post(
|
||||
f"/api/chat/{gid}/personas/{pid}/chat/send",
|
||||
json={"text": "ได้เลยครับ ลองใช้ก่อนได้"},
|
||||
headers=headers,
|
||||
)
|
||||
assert sent.status_code == 200, sent.get_json()
|
||||
body = sent.get_json()
|
||||
|
||||
# Auto-close: the session must be finished with a summary, no manual step.
|
||||
assert body["finished"] is True
|
||||
assert body["session"]["status"] == "finished"
|
||||
assert body["debrief"] is not None
|
||||
assert body["debrief"]["score"] == 55
|
||||
|
||||
# A system note about the trial + coming back must be in the transcript.
|
||||
texts = [m.get("text", "") for m in body["messages"] if m.get("role") == "system"]
|
||||
assert any("ลองใช้" in t or "trial" in t.lower() for t in texts)
|
||||
|
||||
|
||||
def test_buy_decision_auto_closes_won(client, user_store, login, monkeypatch):
|
||||
gid, pid, headers = _ready(client, user_store, login)
|
||||
client.post(
|
||||
f"/api/chat/{gid}/personas/{pid}/chat/start",
|
||||
json={"scenario": "social", "locale": "th", "mode": "preview"},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
stub = StubSim(decision="buy")
|
||||
stub.judge_verdict["outcome"] = "won"
|
||||
monkeypatch.setattr("app.api.chat_routes._sim", lambda g, p: stub)
|
||||
|
||||
sent = client.post(
|
||||
f"/api/chat/{gid}/personas/{pid}/chat/send",
|
||||
json={"text": "โอเค เอาเลยครับ"},
|
||||
headers=headers,
|
||||
)
|
||||
assert sent.status_code == 200, sent.get_json()
|
||||
body = sent.get_json()
|
||||
assert body["finished"] is True
|
||||
assert body["outcome"] == "won"
|
||||
assert body["session"]["status"] == "finished"
|
||||
|
||||
|
||||
def test_walk_decision_auto_closes_lost(client, user_store, login, monkeypatch):
|
||||
gid, pid, headers = _ready(client, user_store, login)
|
||||
client.post(
|
||||
f"/api/chat/{gid}/personas/{pid}/chat/start",
|
||||
json={"scenario": "social", "locale": "th", "mode": "preview"},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
stub = StubSim(decision="walk")
|
||||
stub.judge_verdict["outcome"] = "lost"
|
||||
monkeypatch.setattr("app.api.chat_routes._sim", lambda g, p: stub)
|
||||
|
||||
sent = client.post(
|
||||
f"/api/chat/{gid}/personas/{pid}/chat/send",
|
||||
json={"text": "งั้นไม่เอาดีกว่าครับ"},
|
||||
headers=headers,
|
||||
)
|
||||
assert sent.status_code == 200, sent.get_json()
|
||||
body = sent.get_json()
|
||||
assert body["finished"] is True
|
||||
assert body["outcome"] == "lost"
|
||||
assert body["session"]["status"] == "finished"
|
||||
@@ -124,3 +124,53 @@ def test_prompt_contains_persona_behavior_contract():
|
||||
assert "recontact" in system.lower()
|
||||
assert "wrong_text" in system
|
||||
assert "tolerance" in system.lower()
|
||||
|
||||
|
||||
def test_partial_protocol_without_braces_never_leaks_into_bubble():
|
||||
# Provider appended contract fields as plain text (no braces) — the exact
|
||||
# raw leakage seen in production. Must NOT be rendered verbatim; it must
|
||||
# be rejected so the bounded retry (or fail-closed) path is used.
|
||||
stub = StubLLM([
|
||||
'"สวัสดีค่ะ สนใจสอบถามค่ะ" "decision": "none", "mood": 0',
|
||||
json.dumps({"reply": "สวัสดีค่ะ สนใจสอบถามค่ะ", "decision": "none", "mood": 0}, ensure_ascii=False),
|
||||
])
|
||||
|
||||
reply, meta = _reply(stub)
|
||||
|
||||
assert reply == "สวัสดีค่ะ สนใจสอบถามค่ะ"
|
||||
assert "decision" not in reply
|
||||
assert "mood" not in reply
|
||||
assert len(stub.calls) == 2
|
||||
|
||||
|
||||
def test_malformed_brace_payload_after_retry_fails_closed():
|
||||
stub = StubLLM([
|
||||
'{"reply". "สวัสดีค่ะ" "\'decision": "none", "mood"\' 0}',
|
||||
'{"reply". "ขอลองก่อนนะคะ" "\'decision": "walk"\'}',
|
||||
])
|
||||
|
||||
with pytest.raises(LLMError, match="persona reply"):
|
||||
_reply(stub)
|
||||
|
||||
assert len(stub.calls) == 2
|
||||
|
||||
|
||||
def test_quoted_sentence_unwraps_quotes():
|
||||
stub = StubLLM(['"สวัสดีค่ะ พอดีสนใจสินค้าค่ะ"'])
|
||||
|
||||
reply, _ = _reply(stub)
|
||||
|
||||
assert reply == "สวัสดีค่ะ พอดีสนใจสินค้าค่ะ"
|
||||
assert '"' not in reply
|
||||
|
||||
|
||||
def test_natural_thai_with_colon_is_not_stripped_as_protocol_noise():
|
||||
# A legit spoken reply containing a colon (e.g. a time) must NOT be treated
|
||||
# as trailing protocol noise — the heuristic must not over-strip real text.
|
||||
msg = "เจอกันพรุ่งนี้ 09:00 นะครับ"
|
||||
stub = StubLLM([msg])
|
||||
|
||||
reply, _ = _reply(stub)
|
||||
|
||||
assert reply == msg
|
||||
assert "09:00" in reply
|
||||
|
||||
@@ -24,6 +24,14 @@ filesystem JSON storage (no SQL). i18n TH/EN. No self-registration (admin provis
|
||||
- **user** (trainee) — trains against personas, own board.
|
||||
|
||||
## Current state — local code/security gate passed; production-operation gate pending
|
||||
> **2026-08-18:** live-QA UX fixes landed (committed locally, not yet pushed): persona
|
||||
> replies can no longer leak raw JSON into the chat bubble; all persona openers are natural
|
||||
> greetings (wrong_text cools off only after the seller's first reply); and the chat now
|
||||
> **auto-closes + auto-summarizes** the moment the customer decides (buy / walk / try-to-trial-first),
|
||||
> with the manual "สรุปผล"/Finish button removed from the UI. Full backend suite **336 passed**
|
||||
> (baseline 330 → +6 new tests), frontend build + unit tests clean, independent reviewer passed.
|
||||
> See `docs/engineering-log/2026-08-18-live-qa-ux-fixes.md`. Blocker unchanged: production
|
||||
> operation (runtime cutover, real-provider QA, Redis, audit apply) still requires operator approval.
|
||||
> **2026-08-16:** the S4.4 JSON-importer + error-handler hardening increment (previously staged on top of `dbfce9a`) was re-verified from a clean `requirements.lock.txt` venv (**330 backend tests**, compileall + frontend build clean) and **committed + pushed**. See `docs/engineering-log/2026-08-16-s4-4-importer-errorhandler-commit.md`. Blocker is unchanged: production operation (runtime cutover, real-provider QA, Redis, audit apply) still requires operator approval.
|
||||
|
||||
The current uncommitted remediation is verified on isolated temporary data: **319 backend tests passed** from a clean `requirements.lock.txt` environment, including **166 focused auth/isolation/export/upload regressions**; **4 frontend unit tests** and **12 Playwright fixture journeys** passed across desktop, 320×568, and 500×768; the production frontend build completed with **1,781 modules** and `npm audit` found **0 vulnerabilities**. Compile, AST, diff, dependency, and added-line security checks passed. The checked-in lock is reproducible. The existing local `backend/.venv` has version drift and `pip check` reports the pre-existing `alibabacloud-tea-openapi 0.4.4` versus `cryptography 50.0.0` conflict; `uv pip sync --dry-run` was inspected but not applied. The final fresh exact-current scoped review returned clean five-key verdicts for auth/storage/rate-limit, tenant/group/session isolation, and analytics/export/parser/upload boundaries. JSON stores remain runtime-authoritative; no production operation has been performed.
|
||||
@@ -57,8 +65,9 @@ cd backend && uv run python run.py # Flask :5001
|
||||
- **One-shot:** 1 persona = 1 chat per user; result final (won/lost). `SessionStore` enforces.
|
||||
- **Resume:** unfinished session resumes on re-entry — **no** scenario re-pick (same session+scenario).
|
||||
- **Win/loss = per-turn LLM judge** (`Simulator.evaluate_turn`): every customer reply is evaluated →
|
||||
`{mood, decision(buy|walk|pending), score_delta, reason}`; session ends when decision = buy/walk.
|
||||
**Not** fixed keywords.
|
||||
`{mood, decision(buy|walk|try|pending), score_delta, reason}`; session **auto-ends (close + summary)**
|
||||
when decision = buy/walk/try — no manual "สรุปผล" button (on `try` a system note says the customer
|
||||
will trial first and come back). **Not** fixed keywords.
|
||||
- **2 scenarios only:** `social`, `f2f_call`. Unknown → social.
|
||||
- **Recontact = persona trait** (not a scenario): chats normally, then at turn ≥ 2 a time-lapse
|
||||
system note ("⏳ ผ่านไป 2-3 สัปดาห์…"), then re-engages warmer.
|
||||
|
||||
@@ -66,3 +66,4 @@ Informed by MiroFish (CrowdSight engine) + the hermes-brain-and-tools CrowdSight
|
||||
- `docs/test-evidence/2026-08-15-postgresql-import.md` — temporary-local PostgreSQL importer evidence and target-operation boundary.
|
||||
- `2026-08-15-s4-4-json-import.md` — fail-closed dry-run/apply importer, idempotency, backup, and parity blockers.
|
||||
- `2026-08-16-s4-4-importer-errorhandler-commit.md` — re-verified from clean lock env (330 tests) and committed the staged S4.4 importer + error-handler hardening increment.
|
||||
- `2026-08-18-live-qa-ux-fixes.md` — live-QA UX fixes: persona JSON never leaks into bubble, natural greeting openers (wrong_text cools off only after first reply), auto-close + auto-summarize on buy/walk/try, no manual "สรุปผล" button (336 tests).
|
||||
|
||||
73
docs/engineering-log/2026-08-18-live-qa-ux-fixes.md
Normal file
73
docs/engineering-log/2026-08-18-live-qa-ux-fixes.md
Normal file
@@ -0,0 +1,73 @@
|
||||
# 2026-08-18 — Live-QA UX fixes: JSON leak, complaint openers, auto-close chat
|
||||
|
||||
Date: 2026-08-18
|
||||
Status: implemented + verification evidence; deploy is the repo's normal Gitea→EasyPanel path (operator runs it)
|
||||
|
||||
## Context
|
||||
|
||||
User tested the hardest-tier personas and reported three UX bugs from a live screenshot:
|
||||
|
||||
1. **Raw JSON leaked into the chat bubble.** A persona's reply rendered as
|
||||
`{"reply". "สวัสดีค่ะ...อยากรู้ว่ามีโปรแกรมช่วยจัดการร้านขายเสื้อผ้ามั้ยคะ" "'decision": "none", "mood"' 0}`
|
||||
instead of just the spoken sentence. Root cause: when the provider emits a
|
||||
valid-looking reply **glued to trailing contract keys** (`"decision": ..., "mood": ...`)
|
||||
with or without braces, the old `_parse_persona_reply` fallback could return that
|
||||
non-protocol-looking-but-still-JSON text verbatim as the bubble.
|
||||
2. **Hardest-tier opener was a complaint**, not a greeting ("ไม่มีเงินซื้อหรอก ของแบบนี้แพง" /
|
||||
wrong_text opener `'never mind, forget it'`). The user wants openers to start with
|
||||
"สวัสดี" / "สนใจ" — and the wrong_text persona should open normally, then only cool off
|
||||
("wrong chat") AFTER the seller's first reply.
|
||||
3. **No automatic close / summary + no visible "สรุปผล" button.** Guide mentioned pressing
|
||||
"สรุปผล" but the user expected the chat to auto-close and auto-summarize when the customer
|
||||
decides, with no manual summarize button.
|
||||
|
||||
## Changes
|
||||
|
||||
- `backend/app/services/simulator.py`
|
||||
- Hardened `_parse_persona_reply`: added `_looks_like_protocol()` and
|
||||
`_strip_trailing_protocol_noise()`. Any output carrying the contract's reserved keys
|
||||
(`"reply"`/`"decision"`/`"mood"`, plain or quoted), an opening `{`/`[`/ fenced block, or a
|
||||
quoted fragment + colon is now rejected → bounded retry → fail-closed (`LLMError`), never
|
||||
rendered. Legitimate natural Thai/EN sentences (even with a time colon e.g. "09:00") still
|
||||
pass clean; a lone quoted sentence unwraps to clean text.
|
||||
- Per-turn judge decision vocabulary now includes `try` in addition to `buy`/`walk`/`pending`.
|
||||
- `_special_instr` for `wrong_text`: opener is a normal greeting; ONLY the first reply to the
|
||||
seller's opening may cool off (wrong chat / never mind).
|
||||
- `backend/app/services/persona_prompts.py` — Rule 6 rewritten (wrong_text opens normally, cools
|
||||
off after first reply) + new Rule 8 OPENER RULE: every opener, for every tier incl. wrong_text,
|
||||
must be a natural polite greeting with interest/question — never a complaint/refusal.
|
||||
- `backend/app/api/chat_routes.py` — `send_message` now auto-closes on `buy`/`walk`/`try`. On
|
||||
`try` (customer decides to trial first + come back) it inserts a localized system note
|
||||
("ลูกค้าตัดสินใจจะลองใช้สินค้า/บริการก่อน แล้วจะกลับมาติดต่ออีกครั้ง...") then finalizes
|
||||
(close + judge summary) exactly like buy/walk.
|
||||
- `frontend/src/views/Chat.vue` — removed the manual "สรุปผล"/Finish button, the `finishing`
|
||||
state, and the `finish()` function; chat now auto-closes/summarizes via `chatSend` → `finished`.
|
||||
- `frontend/src/i18n/index.js` — `chatGuideText` (TH + EN) updated: the chat closes automatically
|
||||
when the customer decides (buy / walk / trial first), no instruction to press "สรุปผล".
|
||||
|
||||
## Tests
|
||||
|
||||
- `backend/tests/test_persona_reply_contract.py` +3: partial-protocol-no-braces never leaks
|
||||
(retry then succeed), malformed-brace payload fails closed (`LLMError`), quoted-sentence
|
||||
unwraps quotes.
|
||||
- `backend/tests/test_auto_close_try.py` (new): route-level `chat/send` with a stubbed simulator
|
||||
asserting `try`/`buy`/`walk` all auto-close (`finished=true`, `status=finished`, debrief
|
||||
present) and that `try` inserts a "ลองใช้"/trial system note.
|
||||
|
||||
## Verification evidence
|
||||
|
||||
| Check | Result |
|
||||
|---|---|
|
||||
| Full backend pytest suite (fresh venv via `uv`) | **336 passed** (baseline 330 → +6) |
|
||||
| Frontend `vite build` | clean |
|
||||
| Frontend vitest unit | 4/4 pass |
|
||||
| Static security scan of diff (+lines) | no secrets / shell / eval / pickle / SQL |
|
||||
| Independent reviewer subagent (fail-closed, verified against actual files) | **passed** — parsing is regex/string only, test-fixture credentials recognized as non-secrets |
|
||||
|
||||
## Notes / assumptions
|
||||
|
||||
- The manual backend `chat/finish` endpoint and the `finish`/`finishing`/`finishConfirm` i18n keys
|
||||
are left in place (unused by the UI) as a safe non-breaking safety net; no UI path calls them now.
|
||||
- Sessions the customer has not decided on stay `active` and are resumable (per existing
|
||||
resume-without-re-pick behavior); they only close when a decision is reached.
|
||||
- Deploy remains the repo's normal Gitea→EasyPanel webhook path (≈3 min rebuild) — operator runs it.
|
||||
@@ -81,8 +81,6 @@ const messages = {
|
||||
noPersonas: 'No personas yet',
|
||||
noPersonasHint: 'Personas will appear when analysis finishes.',
|
||||
previewMode: 'Preview mode — admin practice is excluded from trainee analytics.',
|
||||
finishing: 'Finishing…',
|
||||
finishConfirm: 'Finish this practice now? The conversation will be scored and this session will close.',
|
||||
session: 'Session',
|
||||
failurePoints: 'Missed points',
|
||||
coaching: 'Coaching',
|
||||
@@ -155,11 +153,10 @@ const messages = {
|
||||
selectPersona: 'Select a persona to practice',
|
||||
chooseScenario: 'Choose a scenario',
|
||||
chatGuideTitle: 'How to practice',
|
||||
chatGuideText: 'You are playing the salesperson. Chat with this customer and try to close the sale. Ask about their needs, solve their pain, and handle their objections. When you are done, press "Finish & get result" to see your score and coaching.',
|
||||
chatGuideText: 'You are playing the salesperson. Chat with this customer and try to close the sale. Ask about their needs, solve their pain, and handle their objections. When the customer makes a decision (buys, walks away, or chooses to trial first), the chat closes automatically and your score and coaching appear.',
|
||||
chat: 'Chat',
|
||||
start: 'Start',
|
||||
send: 'Send',
|
||||
finish: 'Finish & get result',
|
||||
debrief: 'Result & Coaching',
|
||||
won: 'Won',
|
||||
lost: 'Lost',
|
||||
@@ -258,8 +255,6 @@ const messages = {
|
||||
noPersonas: 'ยังไม่มีบุคคลต้นแบบ',
|
||||
noPersonasHint: 'บุคคลต้นแบบจะปรากฏเมื่อวิเคราะห์กลุ่มสำเร็จ',
|
||||
previewMode: 'โหมด Preview — ผลการฝึกของผู้ดูแลจะไม่เข้า analytics ของผู้ฝึก',
|
||||
finishing: 'กำลังสรุป…',
|
||||
finishConfirm: 'สรุปผลการฝึกตอนนี้หรือไม่ ระบบจะให้คะแนนบทสนทนาและปิด session นี้',
|
||||
session: 'Session',
|
||||
failurePoints: 'จุดที่พลาด',
|
||||
coaching: 'ข้อเสนอแนะ',
|
||||
@@ -332,11 +327,10 @@ const messages = {
|
||||
selectPersona: 'เลือกบุคคลต้นแบบเพื่อฝึก',
|
||||
chooseScenario: 'เลือกสถานการณ์',
|
||||
chatGuideTitle: 'วิธีฝึก',
|
||||
chatGuideText: 'คุณรับบทเป็นพนักงานขาย พูดคุยกับลูกค้าคนนี้เพื่อพยายามปิดการขายให้ได้ สอบถามความต้องการ แก้ไขปัญหาของลูกค้า และรับมือกับข้อโต้แย้ง เมื่อพอใจแล้วกด "สรุปผล" เพื่อดูคะแนนและข้อเสนอแนะ',
|
||||
chatGuideText: 'คุณรับบทเป็นพนักงานขาย พูดคุยกับลูกค้าคนนี้เพื่อพยายามปิดการขายให้ได้ สอบถามความต้องการ แก้ไขปัญหาของลูกค้า และรับมือกับข้อโต้แย้ง เมื่อลูกค้าตัดสินใจ (ซื้อ ปฏิเสธ หรือขอลองใช้ก่อน) ระบบจะปิดแชทและสรุปผลให้อัตโนมัติ',
|
||||
chat: 'แชท',
|
||||
start: 'เริ่มต้น',
|
||||
send: 'ส่ง',
|
||||
finish: 'สรุปผล',
|
||||
debrief: 'ผลลัพธ์และข้อเสนอแนะ',
|
||||
won: 'ปิดการขายได้',
|
||||
lost: 'ปิดการขายไม่ได้',
|
||||
|
||||
@@ -60,7 +60,6 @@
|
||||
<div v-if="phase === 'chat'" class="composer">
|
||||
<input v-model="text" @keyup.enter="send" :disabled="sending" :placeholder="i18n.t('send')" />
|
||||
<button class="primary" @click="send" :disabled="sending || !text.trim()">{{ i18n.t('send') }}</button>
|
||||
<button class="danger" @click="finish" :disabled="sending || finishing">{{ finishing ? i18n.t('finishing') : i18n.t('finish') }}</button>
|
||||
</div>
|
||||
|
||||
<!-- Debrief -->
|
||||
@@ -106,7 +105,6 @@ const phase = ref('pick') // 'pick' | 'chat' | 'done'
|
||||
const messages = ref([])
|
||||
const text = ref('')
|
||||
const sending = ref(false)
|
||||
const finishing = ref(false)
|
||||
const debrief = ref(null)
|
||||
const sessionId = ref(null)
|
||||
const picked = ref('social')
|
||||
@@ -198,25 +196,6 @@ async function send() {
|
||||
}
|
||||
}
|
||||
|
||||
async function finish() {
|
||||
if (finishing.value || sending.value) return
|
||||
if (!window.confirm(i18n.t('finishConfirm'))) return
|
||||
finishing.value = true
|
||||
try {
|
||||
const res = await api.chatFinish(gid, pid)
|
||||
const session = res.session || {}
|
||||
sessionId.value = session.id || sessionId.value
|
||||
messages.value = session.messages || messages.value
|
||||
debrief.value = res.debrief || session.debrief || null
|
||||
phase.value = 'done'
|
||||
scrollDown()
|
||||
} catch (e) {
|
||||
alert(i18n.t('chatActionFailed'))
|
||||
} finally {
|
||||
finishing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
const FIELD_LABELS = {
|
||||
name: 'ชื่อ', tier: 'ระดับ', difficulty: 'ความยาก', profession: 'อาชีพ',
|
||||
age_group: 'ช่วงอายุ', location: 'พื้นที่', income: 'รายได้', budget: 'งบประมาณ',
|
||||
|
||||
Reference in New Issue
Block a user