Files
sales-trainer/backend/app/services/groups.py
Macky 6a2e6a326f fix: restore legacy training data hidden by new visibility schema (migrate-on-read)
Records written before the visibility field existed carry none; the new
fail-closed authorization treated missing visibility as invalid, making every
legacy group unlistable and unreadable. Add resolved_visibility(group) that
derives effective visibility for legacy records only (owner present => private,
absent => public), leaves explicit-malformed visibility fail-closed (None), and
never derives demo/hidden. Apply it at every list, authorization, chat, and
analytics boundary while keeping demo and hidden-preview paths raw and
owner_user_id-based private isolation intact. No persisted data is rewritten.

Backend full suite passes 517; frontend 26/26; production build passes.
2026-08-25 08:59:22 +07:00

789 lines
33 KiB
Python

"""Persona group store: groups hold a sale kit + personas + report.
A group is created by an admin from a setup form + optional files. After analyze,
it contains `personas` (15 by default = 5 per tier) and a `report`. Groups are
editable/re-analyzeable by admins. Trainees only read revealable views of personas
and run one-shot sessions (sessions are stored separately).
"""
from __future__ import annotations
import datetime
from pathlib import Path
from typing import Any
from ..auth.users import is_valid_tenant_id
from ..config import Config
from ..storage.store import JsonStore, new_id
from .store import ensure_persona_shape, revealable_view, validate_persona_traits
DEFAULT_TIERS = ["A", "B", "C"]
PERSONAS_PER_TIER = 5
GROUP_VISIBILITIES = frozenset({"public", "hidden", "private", "demo"})
GROUP_STATUSES = frozenset({"draft", "analyzing", "ready", "failed"})
GROUP_STATUS_TRANSITIONS = {
"draft": frozenset({"draft", "analyzing", "ready", "failed"}),
"analyzing": frozenset({"analyzing", "ready", "failed"}),
"ready": frozenset({"ready", "analyzing", "draft"}),
"failed": frozenset({"failed", "analyzing", "draft"}),
}
GROUP_INPUT_FIELDS = ("product", "segment", "channel", "language")
def group_visibility(value: object) -> str | None:
"""Return only an exact persisted visibility value; malformed is invalid."""
return value if isinstance(value, str) and value in GROUP_VISIBILITIES else None
def resolved_visibility(group: object) -> str | None:
"""Return the effective visibility of a persisted group for authorization.
Records written before the visibility field existed carry no ``visibility``.
The old schema encoded sharing entirely through ``owner_user_id``: a record
with an owner was the owner's private group, and a record without one was a
shared public group. For such legacy records (and only them) we derive the
effective visibility from that historical contract so existing training data
remains listable and accessible without rewriting any persisted data.
A record that carries an explicit but malformed ``visibility`` stays
fail-closed (returns None) like the raw ``group_visibility``. demo/hidden
are never derived from legacy records.
"""
if not isinstance(group, dict):
return None
if "visibility" in group:
return group_visibility(group.get("visibility"))
if "owner_user_id" in group:
return "private"
return "public"
def is_valid_owner_visibility(group: object) -> bool:
"""Return whether the persisted owner/visibility pair is fail-closed.
An owner marker is an authorization boundary, not presentation metadata:
it must always identify a non-empty user and the record must be private.
Conversely, private records without an owner are never treated as shared.
"""
if not isinstance(group, dict):
return False
visibility = resolved_visibility(group)
if "owner_user_id" in group:
owner = group.get("owner_user_id")
return (
isinstance(owner, str)
and bool(owner.strip())
and owner == owner.strip()
and visibility == "private"
)
return visibility in {"public", "hidden", "demo"}
def is_canonical_private_owner(
group: object,
*,
user_id: object,
org_id: object,
) -> bool:
"""Require owner identity and tenant identity to match a private record."""
return (
isinstance(group, dict)
and is_valid_owner_visibility(group)
and "owner_user_id" in group
and isinstance(user_id, str)
and bool(user_id)
and group.get("owner_user_id") == user_id
and is_valid_tenant_id(org_id)
and is_valid_tenant_id(group.get("org_id"))
and group.get("org_id") == org_id
)
def safe_group_input(value: object) -> dict[str, str]:
"""Return only bounded string input fields safe for API summaries."""
if not isinstance(value, dict):
return {}
return {
field: item[:1000]
for field in GROUP_INPUT_FIELDS
if isinstance(item := value.get(field), str)
}
def _canonical_personas(value: object) -> list[dict[str, Any]]:
"""Keep only persisted persona rows that have a stable public id."""
if not isinstance(value, list):
return []
return [
persona
for persona in value
if isinstance(persona, dict)
and isinstance(persona.get("id"), str)
and bool(persona["id"].strip())
]
def _now() -> str:
return datetime.datetime.now(datetime.timezone.utc).isoformat()
def _normalize_personas(value: object) -> list[dict[str, Any]]:
"""Validate persona rows before defaults can hide malformed persisted input."""
if not isinstance(value, list) or not value:
raise ValueError("ready group requires personas")
normalized: list[dict[str, Any]] = []
persona_ids: set[str] = set()
for persona in value:
if (
not isinstance(persona, dict)
or not isinstance(persona.get("id"), str)
or not persona["id"].strip()
):
raise ValueError("persona id is invalid")
shaped = ensure_persona_shape(persona)
try:
validate_persona_traits(shaped)
except (AttributeError, TypeError, ValueError) as exc:
raise ValueError("persona traits are invalid") from exc
shaped["id"] = shaped["id"].strip()
if shaped["id"] in persona_ids:
raise ValueError("persona id must be unique")
persona_ids.add(shaped["id"])
normalized.append(shaped)
return normalized
def validated_personas(value: object) -> list[dict[str, Any]]:
"""Return only a complete, valid persona collection for read paths."""
try:
return _normalize_personas(value)
except (TypeError, ValueError):
return []
def is_ready_group(group: object) -> bool:
"""Return whether a persisted group satisfies the complete ready contract."""
if not isinstance(group, dict) or group.get("status") != "ready":
return False
try:
_validate_ready_components(
sales_kit=group.get("sales_kit"),
report=group.get("report"),
personas=group.get("personas"),
)
except (TypeError, ValueError):
return False
return True
def is_listable_group_state(group: object) -> bool:
"""Reject malformed lifecycle records while preserving valid recovery states."""
if not isinstance(group, dict) or group.get("status") not in GROUP_STATUSES:
return False
return group.get("status") != "ready" or is_ready_group(group)
def _validate_ready_components(
*, sales_kit: object, report: object, personas: object
) -> list[dict[str, Any]]:
"""Return normalized personas only when the complete ready contract is valid."""
normalized = _normalize_personas(personas)
if (
not isinstance(sales_kit, dict)
or not sales_kit
or not isinstance(report, dict)
or not report
):
raise ValueError("ready group requires non-empty sales_kit and report")
return normalized
class GroupStore:
def __init__(self, data_dir: Path) -> None:
self.groups = JsonStore(data_dir / "groups")
def record_lock(self, gid: str):
"""Hold a group transaction lock across analysis work."""
return self.groups.record_lock(gid)
def create(
self,
*,
org_id: str,
creator_id: str,
title: str,
status: str = "draft",
owner_user_id: str | None = None,
visibility: str = "public",
input_data: dict[str, Any] | None = None,
sales_kit: dict[str, Any] | None = None,
report: dict[str, Any] | None = None,
personas: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
if status not in GROUP_STATUSES:
raise ValueError("group status is invalid")
if group_visibility(visibility) is None:
raise ValueError("group visibility is invalid")
if visibility == "demo" and org_id != Config.DEMO_ORG_ID:
raise ValueError("demo groups must use the demo organization")
if owner_user_id is None and visibility == "private":
raise ValueError("private groups require an owner")
if owner_user_id is not None and (
not isinstance(owner_user_id, str)
or not owner_user_id.strip()
or owner_user_id != owner_user_id.strip()
):
raise ValueError("private owner is invalid")
if owner_user_id is not None and visibility == "demo":
raise ValueError("private groups cannot be demo-visible")
normalized_personas = (
[ensure_persona_shape(persona) for persona in personas if isinstance(persona, dict)]
if isinstance(personas, list)
else []
)
if status == "ready":
normalized_personas = _validate_ready_components(
sales_kit=sales_kit,
report=report,
personas=personas,
)
gid = new_id("group")
group = {
"id": gid,
"org_id": org_id,
"creator_id": creator_id,
"title": title or "Untitled group",
"status": status, # draft -> analyzing -> ready | failed
"visibility": visibility, # public | hidden (shared) | private (owner-only)
"created_at": _now(),
"updated_at": _now(),
"analysis_revision": 0,
"input": dict(input_data) if isinstance(input_data, dict) else {},
"sales_kit": sales_kit if isinstance(sales_kit, dict) else None,
"personas": normalized_personas, # full persona dicts
"report": report if isinstance(report, dict) else None,
"error": None,
}
if owner_user_id is not None:
group["owner_user_id"] = owner_user_id
group["visibility"] = "private"
return self.groups.create(group, key=gid)
def get(self, gid: str) -> dict[str, Any]:
return self.groups.get(gid)
def get_or_none(self, gid: str) -> dict[str, Any] | None:
return self.groups.get_or_none(gid)
def update(self, gid: str, **fields: Any) -> dict[str, Any]:
# Every lifecycle/ownership mutation participates in the same cooperative
# record transaction used by the route-level analysis lock. Without this,
# a caller can change ownership/status between a service snapshot and its
# revalidation even when both code paths appear to use record_lock().
with self.record_lock(gid):
current = self.get(gid)
current_status = current.get("status")
if current_status not in GROUP_STATUSES:
raise ValueError("persisted group status is invalid")
requested_status = fields.get("status", current_status)
if requested_status not in GROUP_STATUSES:
raise ValueError("group status is invalid")
if requested_status not in GROUP_STATUS_TRANSITIONS[current_status]:
raise ValueError(
f"group status transition {current_status!r} -> {requested_status!r} is invalid"
)
if "owner_user_id" in fields:
owner_user_id = fields["owner_user_id"]
if (
not isinstance(owner_user_id, str)
or not owner_user_id.strip()
or owner_user_id != owner_user_id.strip()
):
raise ValueError("private owner is invalid")
# Adding an owner marker converts a shared record into a private one;
# never leave the two authorization fields inconsistent.
fields.setdefault("visibility", "private")
visibility = fields.get("visibility", current.get("visibility"))
if group_visibility(visibility) is None:
raise ValueError("group visibility is invalid")
org_id = fields.get("org_id", current.get("org_id"))
owner_present = "owner_user_id" in current or "owner_user_id" in fields
effective_owner = fields.get("owner_user_id", current.get("owner_user_id"))
if owner_present and (
not isinstance(effective_owner, str)
or not effective_owner.strip()
or effective_owner != effective_owner.strip()
):
raise ValueError("private groups require a valid owner")
if visibility == "demo" and org_id != Config.DEMO_ORG_ID:
raise ValueError("demo groups must use the demo organization")
if owner_present and visibility != "private":
raise ValueError("private groups cannot change visibility")
if visibility == "private" and not owner_present:
raise ValueError("private groups require a valid owner")
effective_status = requested_status
raw_personas = fields.get("personas", current.get("personas"))
if effective_status == "ready":
fields["personas"] = _validate_ready_components(
sales_kit=fields.get("sales_kit", current.get("sales_kit")),
report=fields.get("report", current.get("report")),
personas=raw_personas,
)
fields.setdefault("updated_at", _now())
return self.groups.update(gid, **fields)
def publish_analysis(
self,
gid: str,
*,
sales_kit: object,
personas: object,
report: object,
) -> dict[str, Any]:
"""Atomically validate and publish a completed analysis."""
normalized = _validate_ready_components(
sales_kit=sales_kit,
report=report,
personas=personas,
)
with self.record_lock(gid):
current = self.get(gid)
if current.get("status") != "analyzing":
raise ValueError("group status must be analyzing before publication")
published = {
**current,
"sales_kit": sales_kit,
"personas": normalized,
"report": report,
"status": "ready",
"error": None,
"updated_at": _now(),
"analysis_revision": (
current.get("analysis_revision", 0) + 1
if isinstance(current.get("analysis_revision", 0), int)
and not isinstance(current.get("analysis_revision", 0), bool)
and current.get("analysis_revision", 0) >= 0
else 1
),
}
return self.groups.replace(gid, published)
def list_for_org(self, org_id: str | None = None) -> list[dict[str, Any]]:
if not is_valid_tenant_id(org_id):
return []
groups = self.groups.all()
return [g for g in groups if isinstance(g, dict) and g.get("org_id") == org_id]
def list_visible_to(
self,
*,
role: str,
org_id: str | None = None,
user_id: str | None = None,
actor_org_id: str | None = None,
records: list[dict[str, Any]] | None = None,
) -> list[dict[str, Any]]:
"""List groups a role/user can see through a role-safe view.
Trainee callers receive only ready groups, only their own private groups,
and revealable persona fields. Keeping this policy here prevents a new
route from accidentally turning a raw group index into an IDOR/data leak.
"""
groups_all = self.groups.all() if records is None else records
if role == "super_admin":
shared = [
g
for g in groups_all
if (
isinstance(g, dict)
and is_listable_group_state(g)
and is_valid_owner_visibility(g)
and "owner_user_id" not in g
)
]
if org_id is not None:
shared = [g for g in shared if g.get("org_id") == org_id]
owned = [
self._user_visible_group(g, is_owned=True)
for g in groups_all
if (
isinstance(g, dict)
and is_listable_group_state(g)
and is_canonical_private_owner(
g,
user_id=user_id,
org_id=actor_org_id if actor_org_id is not None else org_id,
)
and (org_id is None or g.get("org_id") == org_id)
)
]
return shared + owned
if not is_valid_tenant_id(org_id):
return []
groups = [g for g in groups_all if is_listable_group_state(g)]
groups = [g for g in groups if g.get("org_id") == org_id]
if role == "user":
if not isinstance(user_id, str) or not user_id:
return []
# Own private groups remain visible in every lifecycle state so the
# owner can recover from a failed analysis. Shared groups stay
# public-and-ready only. Malformed visibility is never public.
groups = [
g
for g in groups
if (
is_valid_owner_visibility(g)
and "owner_user_id" in g
and g.get("owner_user_id") == user_id
and resolved_visibility(g) == "private"
)
or (
is_valid_owner_visibility(g)
and "owner_user_id" not in g
and is_ready_group(g)
and resolved_visibility(g) == "public"
)
]
return [
self._user_visible_group(
g,
is_owned=g.get("owner_user_id") == user_id,
)
for g in groups
]
if role == "demo":
if org_id != Config.DEMO_ORG_ID or not isinstance(user_id, str) or not user_id:
return []
groups = [
g
for g in groups
if (
is_ready_group(g)
and group_visibility(g.get("visibility")) == "demo"
and "owner_user_id" not in g
)
]
return [self._user_visible_group(g, is_owned=False) for g in groups]
if role == "admin":
# A tenant admin may inspect shared group summaries (incl. hidden/draft),
# never a trainee's private group. An explicit owner marker is private
# even when malformed; malformed visibility is not a shared summary.
shared = [
g
for g in groups
if (
is_valid_owner_visibility(g)
and "owner_user_id" not in g
and resolved_visibility(g) in {"public", "hidden"}
)
]
owned = [
self._user_visible_group(g, is_owned=True)
for g in groups
if (
isinstance(user_id, str)
and user_id
and is_valid_owner_visibility(g)
and g.get("owner_user_id") == user_id
)
]
return shared + owned
return []
@staticmethod
def _user_visible_group(
group: dict[str, Any],
*,
is_owned: bool,
) -> dict[str, Any]:
"""Return the closed group envelope allowed to a trainee."""
source = safe_group_input(group.get("input"))
status = group.get("status", "draft")
personas = validated_personas(group.get("personas")) if status == "ready" else []
return {
"id": group.get("id"),
"org_id": group.get("org_id"),
"visibility": group.get("visibility", "public"),
"is_owned": is_owned,
"title": group.get("title", ""),
"status": status,
"created_at": group.get("created_at"),
"updated_at": group.get("updated_at"),
"input": source,
"sales_kit": None,
"personas": [
revealable_view(ensure_persona_shape(persona))
for persona in personas
],
"report": None,
"error": "analysis_failed" if status == "failed" and group.get("error") else None,
}
def get_or_create_private_group(
self,
*,
org_id: str,
owner_user_id: str,
owner_name: str = "User",
input_data: dict[str, Any] | None = None,
sales_kit: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Return the one private group owned by a trainee.
The lookup and create are one invariant. Keep both inside the
collection lock so concurrent web workers cannot allocate duplicate
private groups for the same ``(org_id, owner_user_id)`` pair. Candidate
records are re-read under their record lock after the collection snapshot
so ownership/status changes cannot make this return stale authorization
state.
"""
if not is_valid_tenant_id(org_id):
raise ValueError("organization scope is invalid")
if (
not isinstance(owner_user_id, str)
or not owner_user_id.strip()
or owner_user_id != owner_user_id.strip()
):
raise ValueError("private owner is invalid")
for _ in range(3):
candidate_id = None
found_candidate = False
with self.groups.collection_lock():
for group in self.list_for_org(org_id=org_id):
if group.get("owner_user_id") == owner_user_id:
found_candidate = True
candidate_id = group.get("id")
break
if not found_candidate:
# An empty private group is a recoverable draft, never a
# ready/publication state. The first persona is published by
# set_personas() or append_private_persona().
return self.create(
org_id=org_id,
creator_id=owner_user_id,
title=f"{owner_name or 'User'}'s private personas",
status="draft",
owner_user_id=owner_user_id,
input_data=input_data or {"channel": "social", "language": "th"},
sales_kit=sales_kit or {
"productName": "personal practice",
"valueProps": [],
"features": [],
},
)
if (
not isinstance(candidate_id, str)
or not candidate_id.strip()
or candidate_id != candidate_id.strip()
):
raise ValueError("private group state is invalid")
with self.record_lock(candidate_id):
current = self.get_or_none(candidate_id)
if current is None:
continue
if (
current.get("org_id") != org_id
or current.get("owner_user_id") != owner_user_id
):
continue
if not is_valid_owner_visibility(current):
raise ValueError("private group state is invalid")
if current.get("status") not in {"draft", "analyzing", "ready", "failed"}:
raise ValueError("private group state is invalid")
status = current.get("status")
if status == "ready":
try:
normalized = _validate_ready_components(
sales_kit=current.get("sales_kit"),
report=current.get("report"),
personas=current.get("personas"),
)
except (TypeError, ValueError):
# A malformed persisted ready record is recoverable, but
# must never be trusted or exposed as ready.
return self.update(
candidate_id,
status="draft",
personas=[],
)
if normalized != current.get("personas"):
current = self.update(candidate_id, personas=normalized)
return current
if status == "draft":
raw_personas = current.get("personas")
if raw_personas:
try:
_normalize_personas(raw_personas)
except (TypeError, ValueError):
return self.update(candidate_id, personas=[])
raise ValueError("private group is not ready")
return current
raise ValueError("private group is not ready")
raise ValueError("private group state is unavailable")
def append_private_persona(
self,
*,
org_id: str,
owner_user_id: str,
owner_name: str = "User",
persona: dict[str, Any],
input_data: dict[str, Any] | None = None,
sales_kit: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Append one persona without publishing a ready-empty private group.
A new private group is created with its first persona in the same
persisted record. Existing groups are re-read and updated under their
record lock. The collection lock covers lookup/create so concurrent
variant requests cannot race into duplicate private groups.
"""
if not is_valid_tenant_id(org_id):
raise ValueError("organization scope is invalid")
if (
not isinstance(owner_user_id, str)
or not owner_user_id.strip()
or owner_user_id != owner_user_id.strip()
):
raise ValueError("private owner is invalid")
if not isinstance(persona, dict):
raise ValueError("persona is invalid")
if not isinstance(persona.get("id"), str) or not persona["id"].strip():
raise ValueError("persona id is invalid")
normalized = ensure_persona_shape(persona)
validate_persona_traits(normalized)
for _ in range(3):
candidate_id = None
found_candidate = False
with self.groups.collection_lock():
for group in self.list_for_org(org_id=org_id):
if group.get("owner_user_id") == owner_user_id:
found_candidate = True
candidate_id = group.get("id")
break
if not found_candidate:
return self.create(
org_id=org_id,
creator_id=owner_user_id,
title=f"{owner_name or 'User'}'s private personas",
status="ready",
owner_user_id=owner_user_id,
input_data=input_data or {"channel": "social", "language": "th"},
sales_kit=sales_kit or {
"productName": "personal practice",
"valueProps": [],
"features": [],
},
report={"type": "private_persona_collection"},
personas=[normalized],
)
if (
not isinstance(candidate_id, str)
or not candidate_id.strip()
or candidate_id != candidate_id.strip()
):
raise ValueError("private group state is invalid")
with self.record_lock(candidate_id):
current = self.get_or_none(candidate_id)
if current is None:
continue
if (
current.get("org_id") != org_id
or current.get("owner_user_id") != owner_user_id
):
continue
if not is_valid_owner_visibility(current):
raise ValueError("private group state is invalid")
raw_personas = current.get("personas")
if raw_personas:
try:
current_personas = _normalize_personas(raw_personas)
except (TypeError, ValueError) as exc:
raise ValueError("private group personas are invalid") from exc
else:
current_personas = []
if current.get("status") == "draft" and current_personas:
raise ValueError("private group is not ready")
if current.get("status") not in {"draft", "ready"}:
raise ValueError("private group is not ready")
fields: dict[str, Any] = {
"personas": current_personas + [normalized],
"status": "ready",
}
if not isinstance(current.get("report"), dict) or not current.get("report"):
fields["report"] = {"type": "private_persona_collection"}
if not isinstance(current.get("sales_kit"), dict) or not current.get("sales_kit"):
fields["sales_kit"] = sales_kit or {
"productName": "personal practice",
"valueProps": [],
"features": [],
}
return self.update(candidate_id, **fields)
raise ValueError("private group state is unavailable")
# ── personas ────────────────────────────────────────────────────────
def set_personas(self, gid: str, personas: list[dict[str, Any]]) -> dict[str, Any]:
if not isinstance(personas, list) or any(
not isinstance(persona, dict) for persona in personas
):
raise ValueError("personas must be a list of objects")
normalized = _normalize_personas(personas)
with self.record_lock(gid):
current = self.get(gid)
fields: dict[str, Any] = {"personas": normalized, "status": "ready"}
if current.get("owner_user_id"):
if not isinstance(current.get("report"), dict) or not current.get("report"):
fields["report"] = {"type": "private_persona_collection"}
if not isinstance(current.get("sales_kit"), dict) or not current.get("sales_kit"):
fields["sales_kit"] = {
"productName": "personal practice",
"valueProps": [],
"features": [],
}
return self.update(gid, **fields)
def get_persona(self, gid: str, pid: str) -> dict[str, Any] | None:
group = self.get(gid)
raw_personas = group.get("personas")
personas = raw_personas if isinstance(raw_personas, list) else []
for p in personas:
if not isinstance(p, dict):
continue
if p.get("id") == pid:
return p
return None
def update_persona(self, gid: str, pid: str, patch: dict[str, Any]) -> dict[str, Any]:
with self.record_lock(gid):
group = self.get(gid)
if group.get("status") != "ready":
raise ValueError("ready group required")
found = False
raw_personas = group.get("personas")
personas = raw_personas if isinstance(raw_personas, list) else []
group["personas"] = personas
for i, p in enumerate(personas):
if not isinstance(p, dict):
continue
if p.get("id") == pid:
merged = {**p, **patch, "id": pid}
normalized = ensure_persona_shape(merged)
validate_persona_traits(normalized)
group["personas"][i] = normalized
found = True
break
if not found:
raise ValueError("persona not found")
if not isinstance(group.get("sales_kit"), dict) or not group.get("sales_kit"):
raise ValueError("ready group sales kit is invalid")
if not isinstance(group.get("report"), dict) or not group.get("report"):
raise ValueError("ready group report is invalid")
return self.groups.replace(gid, group)
def delete(self, gid: str) -> None:
"""Hard-delete a group (personas/report included)."""
with self.record_lock(gid):
self.groups.delete(gid)