feat: SaaS foundation for CrowdSight
Elevate MiroFish/CrowdSight from single-container dev to a SaaS foundation: - Local memory backend (Zep-compatible): memory services/models, local graph builder + updater, AgentActivity seam, import-boundary isolation; Zep stays default, local is opt-in behind MEMORY_BACKEND. Semantic parity not yet proven. - Durable product persistence: projects/simulations/reports schema (migration 0007) + tenant/owner-scoped ProductRepository + dual-write + scoped_project read-first + ArtifactStore abstraction; durable JobQueue + worker.py. - SaaS hardening: durable RateLimiter (wired to login), UsageService (LLM accounting), redacted AuditService, idempotency, CORS allowlist, safe API errors, single-use PasswordResetService + endpoints (covers invite-pending). - Exactly 3 roles (super_admin/admin/user) with tenant authz policy. - Admin UI: GET/POST/PATCH /api/admin/users + GET/PUT /api/admin/settings (super-admin only, encrypted/masked); AdminView.vue + SettingsView.vue with admin/super-admin route guards, th/en i18n. - Production deploy topology: multi-stage Dockerfile (frontend build + gunicorn wsgi + nginx SPA-proxy + supervisord worker), backend/wsgi.py, gunicorn dep. Backend 197 passed; frontend 10 tests + build green. ruff unavailable (gap). No commit of credentials; secrets handled via env/.env.example. Deferred: Zep semantic A/B parity, object storage cutover, mobile QA, EasyPanel container build of deploy topology.
This commit is contained in:
Kunthawat Greethong
@@ -4,6 +4,21 @@
|
||||
|
||||
from .task import TaskManager, TaskStatus
|
||||
from .project import Project, ProjectStatus, ProjectManager
|
||||
from .saas import AuthSession, Membership, Organization, User
|
||||
from .memory import MemoryEdge, MemoryEpisode, MemoryGraph, MemoryNode
|
||||
from .operations import AuditLog, IdempotencyRecord, Job, JobStatus
|
||||
from .product import DurableReport, ProductProject, ProductSimulation, ProjectStatus, ReportStatus, SimulationStatus
|
||||
from .settings import PlatformSettings
|
||||
from .rate_limit import RateLimitEvent
|
||||
from .usage import UsageEvent
|
||||
from .password_reset import PasswordResetToken
|
||||
|
||||
__all__ = ['TaskManager', 'TaskStatus', 'Project', 'ProjectStatus', 'ProjectManager']
|
||||
__all__ = [
|
||||
'TaskManager', 'TaskStatus', 'Project', 'ProjectStatus', 'ProjectManager',
|
||||
'AuthSession', 'Membership', 'Organization', 'User',
|
||||
'MemoryEdge', 'MemoryEpisode', 'MemoryGraph', 'MemoryNode',
|
||||
'AuditLog', 'IdempotencyRecord', 'Job', 'JobStatus',
|
||||
'ProductProject', 'ProductSimulation', 'DurableReport', 'SimulationStatus', 'ReportStatus',
|
||||
'PlatformSettings', 'RateLimitEvent', 'UsageEvent', 'PasswordResetToken',
|
||||
]
|
||||
|
||||
|
||||
141
backend/app/models/memory.py
Normal file
141
backend/app/models/memory.py
Normal file
@@ -0,0 +1,141 @@
|
||||
"""Durable local graph-memory schema replacing the storage side of Zep."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import DateTime, Float, ForeignKey, Index, JSON, String, Text, UniqueConstraint, func, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from ..db import Base
|
||||
|
||||
|
||||
def _id(prefix: str) -> str:
|
||||
return f"{prefix}_{uuid4().hex}"
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class MemoryGraph(Base):
|
||||
__tablename__ = "memory_graphs"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(128), primary_key=True, default=lambda: _id("graph"))
|
||||
organization_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
project_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
||||
ontology: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict, server_default=text("'{}'"))
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="ready", server_default=text("'ready'"))
|
||||
version: Mapped[int] = mapped_column(nullable=False, default=1, server_default=text("1"))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now(), onupdate=_utc_now
|
||||
)
|
||||
|
||||
episodes: Mapped[list["MemoryEpisode"]] = relationship(
|
||||
back_populates="graph", cascade="all, delete-orphan"
|
||||
)
|
||||
nodes: Mapped[list["MemoryNode"]] = relationship(
|
||||
back_populates="graph", cascade="all, delete-orphan"
|
||||
)
|
||||
edges: Mapped[list["MemoryEdge"]] = relationship(
|
||||
back_populates="graph", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
|
||||
class MemoryEpisode(Base):
|
||||
__tablename__ = "memory_episodes"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("graph_id", "source_type", "source_ref", name="uq_memory_episode_source"),
|
||||
Index("ix_memory_episodes_graph_status", "graph_id", "status"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(128), primary_key=True, default=lambda: _id("episode"))
|
||||
graph_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("memory_graphs.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
source_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
source_ref: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
normalized_text: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
summary: Mapped[str] = mapped_column(Text, nullable=False, default="", server_default=text("''"))
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="processed", server_default=text("'processed'"))
|
||||
extractor_version: Mapped[str] = mapped_column(String(64), nullable=False, default="v1", server_default=text("'v1'"))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now()
|
||||
)
|
||||
|
||||
graph: Mapped[MemoryGraph] = relationship(back_populates="episodes")
|
||||
|
||||
|
||||
class MemoryNode(Base):
|
||||
__tablename__ = "memory_nodes"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("graph_id", "normalized_name", name="uq_memory_node_graph_name"),
|
||||
Index("ix_memory_nodes_graph_name", "graph_id", "normalized_name"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(128), primary_key=True, default=lambda: _id("node"))
|
||||
graph_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("memory_graphs.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
canonical_name: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
normalized_name: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
labels: Mapped[list] = mapped_column(JSON, nullable=False, default=list, server_default=text("'[]'"))
|
||||
aliases: Mapped[list] = mapped_column(JSON, nullable=False, default=list, server_default=text("'[]'"))
|
||||
attributes: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict, server_default=text("'{}'"))
|
||||
summary: Mapped[str] = mapped_column(Text, nullable=False, default="", server_default=text("''"))
|
||||
confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.0, server_default=text("0"))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now(), onupdate=_utc_now
|
||||
)
|
||||
|
||||
graph: Mapped[MemoryGraph] = relationship(back_populates="nodes")
|
||||
outgoing_edges: Mapped[list["MemoryEdge"]] = relationship(
|
||||
foreign_keys="MemoryEdge.source_node_id", back_populates="source_node"
|
||||
)
|
||||
incoming_edges: Mapped[list["MemoryEdge"]] = relationship(
|
||||
foreign_keys="MemoryEdge.target_node_id", back_populates="target_node"
|
||||
)
|
||||
|
||||
|
||||
class MemoryEdge(Base):
|
||||
__tablename__ = "memory_edges"
|
||||
__table_args__ = (
|
||||
Index("ix_memory_edges_graph_relation", "graph_id", "relation"),
|
||||
Index("ix_memory_edges_graph_temporal", "graph_id", "valid_at", "invalid_at"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(128), primary_key=True, default=lambda: _id("edge"))
|
||||
graph_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("memory_graphs.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
source_node_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("memory_nodes.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
target_node_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("memory_nodes.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
relation: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
fact: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
attributes: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict, server_default=text("'{}'"))
|
||||
confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.0, server_default=text("0"))
|
||||
valid_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
invalid_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
expired_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now()
|
||||
)
|
||||
|
||||
graph: Mapped[MemoryGraph] = relationship(back_populates="edges")
|
||||
source_node: Mapped[MemoryNode] = relationship(
|
||||
foreign_keys=[source_node_id], back_populates="outgoing_edges"
|
||||
)
|
||||
target_node: Mapped[MemoryNode] = relationship(
|
||||
foreign_keys=[target_node_id], back_populates="incoming_edges"
|
||||
)
|
||||
128
backend/app/models/operations.py
Normal file
128
backend/app/models/operations.py
Normal file
@@ -0,0 +1,128 @@
|
||||
"""Durable operation metadata for jobs, retries, idempotency, and audit."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import JSON, CheckConstraint, DateTime, ForeignKey, Index, Integer, String, Text, UniqueConstraint, func, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from ..db import Base
|
||||
|
||||
|
||||
def _id(prefix: str) -> str:
|
||||
return f"{prefix}_{uuid4().hex}"
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class JobStatus(str, Enum):
|
||||
QUEUED = "queued"
|
||||
RUNNING = "running"
|
||||
SUCCEEDED = "succeeded"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
class Job(Base):
|
||||
"""Durable, tenant-owned unit of asynchronous work."""
|
||||
|
||||
__tablename__ = "jobs"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"status IN ('queued', 'running', 'succeeded', 'failed', 'cancelled')",
|
||||
name="ck_jobs_status",
|
||||
),
|
||||
Index("ix_jobs_org_status_created", "organization_id", "status", "created_at"),
|
||||
Index("ix_jobs_org_owner", "organization_id", "owner_user_id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: _id("job"))
|
||||
organization_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
owner_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
project_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
|
||||
graph_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
|
||||
operation: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||
status: Mapped[JobStatus] = mapped_column(
|
||||
String(32), nullable=False, default=JobStatus.QUEUED.value, server_default=text("'queued'")
|
||||
)
|
||||
progress: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
|
||||
message: Mapped[str] = mapped_column(Text, nullable=False, default="", server_default=text("''"))
|
||||
result: Mapped[dict | list | None] = mapped_column(JSON, nullable=True)
|
||||
progress_detail: Mapped[dict | list | None] = mapped_column(JSON, nullable=True)
|
||||
job_metadata: Mapped[dict | list | None] = mapped_column("metadata", JSON, nullable=True)
|
||||
error_code: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
result_ref: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
|
||||
settings_version: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=_utc_now, server_default=func.now(), nullable=False
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=_utc_now, onupdate=_utc_now, server_default=func.now(), nullable=False
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class IdempotencyRecord(Base):
|
||||
"""Request fingerprint and replayable response for retry-safe mutations."""
|
||||
|
||||
__tablename__ = "idempotency_records"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("organization_id", "user_id", "key", name="uq_idempotency_org_user_key"),
|
||||
Index("ix_idempotency_expiry", "expires_at"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: _id("idem"))
|
||||
organization_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
user_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
key: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
request_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="reserved", server_default=text("'reserved'")
|
||||
)
|
||||
response_status: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
response_body: Mapped[dict | list | None] = mapped_column(JSON, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=_utc_now, server_default=func.now(), nullable=False
|
||||
)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class AuditLog(Base):
|
||||
"""Tenant-scoped redacted audit event; details must never contain secrets."""
|
||||
|
||||
__tablename__ = "audit_logs"
|
||||
__table_args__ = (
|
||||
Index("ix_audit_org_created", "organization_id", "created_at"),
|
||||
Index("ix_audit_org_target", "organization_id", "target_type", "target_id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: _id("audit"))
|
||||
organization_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
actor_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
action: Mapped[str] = mapped_column(String(160), nullable=False)
|
||||
target_type: Mapped[str] = mapped_column(String(80), nullable=False)
|
||||
target_id: Mapped[str | None] = mapped_column(String(160), nullable=True)
|
||||
details: Mapped[dict | list | None] = mapped_column("metadata", JSON, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=_utc_now, server_default=func.now(), nullable=False
|
||||
)
|
||||
37
backend/app/models/password_reset.py
Normal file
37
backend/app/models/password_reset.py
Normal file
@@ -0,0 +1,37 @@
|
||||
"""Durable, single-use password reset tokens."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, ForeignKey, String, func, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from ..db import Base
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class PasswordResetToken(Base):
|
||||
"""One hash of a one-time reset token; plaintext is never stored."""
|
||||
|
||||
__tablename__ = "password_reset_tokens"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(64), primary_key=True, default=lambda: f"prt_{uuid4().hex}"
|
||||
)
|
||||
user_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
token_hash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
auth_version: Mapped[int] = mapped_column(nullable=False, default=0, server_default=text("0"))
|
||||
used: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default=text("0")
|
||||
)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now()
|
||||
)
|
||||
186
backend/app/models/product.py
Normal file
186
backend/app/models/product.py
Normal file
@@ -0,0 +1,186 @@
|
||||
"""Durable product-resource schema: projects, simulations, and reports.
|
||||
|
||||
These replace the legacy filesystem-backed ProjectManager / SimulationManager /
|
||||
ReportManager payloads with tenant- and owner-scoped SQL rows, so product state
|
||||
survives restarts and multiple workers without cross-tenant leakage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import (
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from ..db import Base
|
||||
|
||||
|
||||
def _id(prefix: str) -> str:
|
||||
return f"{prefix}_{uuid4().hex}"
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class ProjectStatus(str, Enum):
|
||||
CREATED = "created"
|
||||
ONTOLOGY_GENERATED = "ontology_generated"
|
||||
GRAPH_BUILDING = "graph_building"
|
||||
GRAPH_COMPLETED = "graph_completed"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class SimulationStatus(str, Enum):
|
||||
CREATED = "created"
|
||||
PREPARING = "preparing"
|
||||
READY = "ready"
|
||||
RUNNING = "running"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
class ReportStatus(str, Enum):
|
||||
DRAFT = "draft"
|
||||
PLANNING = "planning"
|
||||
GENERATING = "generating"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class ProductProject(Base):
|
||||
"""Durable, tenant-owned project record (metadata + ontology)."""
|
||||
|
||||
__tablename__ = "projects"
|
||||
__table_args__ = (
|
||||
Index("ix_projects_org_owner", "organization_id", "owner_user_id"),
|
||||
Index("ix_projects_org_created", "organization_id", "created_at"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(128), primary_key=True, default=lambda: _id("project")
|
||||
)
|
||||
organization_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
owner_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False, server_default=text("''"))
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default=ProjectStatus.CREATED.value, server_default=text("'created'")
|
||||
)
|
||||
language: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="en", server_default=text("'en'")
|
||||
)
|
||||
total_text_length: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0, server_default=text("0")
|
||||
)
|
||||
source_metadata: Mapped[dict | list | None] = mapped_column(JSON, nullable=True)
|
||||
ontology: Mapped[dict | list | None] = mapped_column(JSON, nullable=True)
|
||||
analysis_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
simulation_requirement: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
graph_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
|
||||
graph_build_task_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now(), onupdate=_utc_now
|
||||
)
|
||||
|
||||
|
||||
class ProductSimulation(Base):
|
||||
"""Durable, tenant-scoped simulation with a config snapshot."""
|
||||
|
||||
__tablename__ = "simulations"
|
||||
__table_args__ = (
|
||||
Index("ix_simulations_org_project", "organization_id", "project_id"),
|
||||
Index("ix_simulations_org_created", "organization_id", "created_at"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(128), primary_key=True, default=lambda: _id("sim")
|
||||
)
|
||||
organization_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
project_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("projects.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default=SimulationStatus.CREATED.value, server_default=text("'created'")
|
||||
)
|
||||
platform: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="parallel", server_default=text("'parallel'")
|
||||
)
|
||||
config: Mapped[dict | list | None] = mapped_column(JSON, nullable=True)
|
||||
current_round: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now(), onupdate=_utc_now
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class DurableReport(Base):
|
||||
"""Durable, tenant-scoped report with outline/status/content metadata."""
|
||||
|
||||
__tablename__ = "reports"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("organization_id", "id", name="uq_reports_org_id"),
|
||||
Index("ix_reports_org_project", "organization_id", "project_id"),
|
||||
Index("ix_reports_org_simulation", "organization_id", "simulation_id"),
|
||||
Index("ix_reports_org_created", "organization_id", "created_at"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(128), primary_key=True, default=lambda: _id("report")
|
||||
)
|
||||
organization_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
project_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("projects.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
simulation_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("simulations.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default=ReportStatus.DRAFT.value, server_default=text("'draft'")
|
||||
)
|
||||
title: Mapped[str] = mapped_column(String(255), nullable=False, server_default=text("''"))
|
||||
outline: Mapped[dict | list | None] = mapped_column(JSON, nullable=True)
|
||||
markdown_content: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now(), onupdate=_utc_now
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
@@ -5,8 +5,10 @@
|
||||
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
import shutil
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, List, Optional
|
||||
from enum import Enum
|
||||
@@ -31,6 +33,8 @@ class Project:
|
||||
status: ProjectStatus
|
||||
created_at: str
|
||||
updated_at: str
|
||||
organization_id: Optional[str] = None
|
||||
owner_user_id: Optional[str] = None
|
||||
|
||||
# 文件信息
|
||||
files: List[Dict[str, str]] = field(default_factory=list) # [{filename, path, size}]
|
||||
@@ -60,6 +64,8 @@ class Project:
|
||||
"status": self.status.value if isinstance(self.status, ProjectStatus) else self.status,
|
||||
"created_at": self.created_at,
|
||||
"updated_at": self.updated_at,
|
||||
"organization_id": self.organization_id,
|
||||
"owner_user_id": self.owner_user_id,
|
||||
"files": self.files,
|
||||
"total_text_length": self.total_text_length,
|
||||
"ontology": self.ontology,
|
||||
@@ -85,6 +91,8 @@ class Project:
|
||||
status=status,
|
||||
created_at=data.get('created_at', ''),
|
||||
updated_at=data.get('updated_at', ''),
|
||||
organization_id=data.get('organization_id'),
|
||||
owner_user_id=data.get('owner_user_id'),
|
||||
files=data.get('files', []),
|
||||
total_text_length=data.get('total_text_length', 0),
|
||||
ontology=data.get('ontology'),
|
||||
@@ -100,6 +108,8 @@ class Project:
|
||||
|
||||
class ProjectManager:
|
||||
"""项目管理器 - 负责项目的持久化存储和检索"""
|
||||
|
||||
_SAFE_PROJECT_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
|
||||
|
||||
# 项目存储根目录
|
||||
PROJECTS_DIR = os.path.join(Config.UPLOAD_FOLDER, 'projects')
|
||||
@@ -109,10 +119,21 @@ class ProjectManager:
|
||||
"""确保项目目录存在"""
|
||||
os.makedirs(cls.PROJECTS_DIR, exist_ok=True)
|
||||
|
||||
@classmethod
|
||||
def _validate_project_id(cls, project_id: str) -> str:
|
||||
if not isinstance(project_id, str) or not cls._SAFE_PROJECT_ID.fullmatch(project_id):
|
||||
raise ValueError("invalid_project_id")
|
||||
return project_id
|
||||
|
||||
@classmethod
|
||||
def _get_project_dir(cls, project_id: str) -> str:
|
||||
"""获取项目目录路径"""
|
||||
return os.path.join(cls.PROJECTS_DIR, project_id)
|
||||
"""获取项目目录路径，拒绝路径分隔符和 traversal。"""
|
||||
safe_project_id = cls._validate_project_id(project_id)
|
||||
root = os.path.realpath(cls.PROJECTS_DIR)
|
||||
project_dir = os.path.realpath(os.path.join(root, safe_project_id))
|
||||
if os.path.commonpath([root, project_dir]) != root:
|
||||
raise ValueError("invalid_project_id")
|
||||
return project_dir
|
||||
|
||||
@classmethod
|
||||
def _get_project_meta_path(cls, project_id: str) -> str:
|
||||
@@ -130,7 +151,12 @@ class ProjectManager:
|
||||
return os.path.join(cls._get_project_dir(project_id), 'extracted_text.txt')
|
||||
|
||||
@classmethod
|
||||
def create_project(cls, name: str = "Unnamed Project") -> Project:
|
||||
def create_project(
|
||||
cls,
|
||||
name: str = "Unnamed Project",
|
||||
organization_id: Optional[str] = None,
|
||||
owner_user_id: Optional[str] = None,
|
||||
) -> Project:
|
||||
"""
|
||||
创建新项目
|
||||
|
||||
@@ -150,7 +176,9 @@ class ProjectManager:
|
||||
name=name,
|
||||
status=ProjectStatus.CREATED,
|
||||
created_at=now,
|
||||
updated_at=now
|
||||
updated_at=now,
|
||||
organization_id=organization_id,
|
||||
owner_user_id=owner_user_id,
|
||||
)
|
||||
|
||||
# 创建项目目录结构
|
||||
@@ -166,12 +194,24 @@ class ProjectManager:
|
||||
|
||||
@classmethod
|
||||
def save_project(cls, project: Project) -> None:
|
||||
"""保存项目元数据"""
|
||||
"""保存项目元数据，使用同目录临时文件+原子替换。"""
|
||||
project.updated_at = datetime.now().isoformat()
|
||||
project_dir = cls._get_project_dir(project.project_id)
|
||||
os.makedirs(project_dir, exist_ok=True)
|
||||
meta_path = cls._get_project_meta_path(project.project_id)
|
||||
|
||||
with open(meta_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(project.to_dict(), f, ensure_ascii=False, indent=2)
|
||||
fd, temp_path = tempfile.mkstemp(prefix=".project-", suffix=".json", dir=project_dir)
|
||||
try:
|
||||
with os.fdopen(fd, 'w', encoding='utf-8') as f:
|
||||
json.dump(project.to_dict(), f, ensure_ascii=False, indent=2)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(temp_path, meta_path)
|
||||
except Exception:
|
||||
try:
|
||||
os.unlink(temp_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
raise
|
||||
|
||||
@classmethod
|
||||
def get_project(cls, project_id: str) -> Optional[Project]:
|
||||
@@ -184,18 +224,32 @@ class ProjectManager:
|
||||
Returns:
|
||||
Project对象，如果不存在返回None
|
||||
"""
|
||||
meta_path = cls._get_project_meta_path(project_id)
|
||||
|
||||
try:
|
||||
meta_path = cls._get_project_meta_path(project_id)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
if not os.path.exists(meta_path):
|
||||
return None
|
||||
|
||||
with open(meta_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
try:
|
||||
with open(meta_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
except (OSError, json.JSONDecodeError, TypeError, KeyError, ValueError):
|
||||
return None
|
||||
|
||||
return Project.from_dict(data)
|
||||
try:
|
||||
return Project.from_dict(data)
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def list_projects(cls, limit: int = 50) -> List[Project]:
|
||||
def list_projects(
|
||||
cls,
|
||||
limit: int = 50,
|
||||
organization_id: Optional[str] = None,
|
||||
owner_user_id: Optional[str] = None,
|
||||
) -> List[Project]:
|
||||
"""
|
||||
列出所有项目
|
||||
|
||||
@@ -210,30 +264,84 @@ class ProjectManager:
|
||||
projects = []
|
||||
for project_id in os.listdir(cls.PROJECTS_DIR):
|
||||
project = cls.get_project(project_id)
|
||||
if project:
|
||||
projects.append(project)
|
||||
if not project:
|
||||
continue
|
||||
if organization_id is not None and project.organization_id != organization_id:
|
||||
continue
|
||||
if owner_user_id is not None and project.owner_user_id != owner_user_id:
|
||||
continue
|
||||
projects.append(project)
|
||||
|
||||
# 按创建时间倒序排序
|
||||
projects.sort(key=lambda p: p.created_at, reverse=True)
|
||||
|
||||
return projects[:limit]
|
||||
|
||||
@classmethod
|
||||
def find_project_by_graph_id(
|
||||
cls,
|
||||
graph_id: str,
|
||||
*,
|
||||
organization_id: str,
|
||||
owner_user_id: Optional[str] = None,
|
||||
) -> Optional[Project]:
|
||||
for project in cls.list_projects(
|
||||
organization_id=organization_id,
|
||||
owner_user_id=owner_user_id,
|
||||
):
|
||||
if project.graph_id == graph_id:
|
||||
return project
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def get_project_for_scope(
|
||||
cls,
|
||||
project_id: str,
|
||||
*,
|
||||
organization_id: str,
|
||||
owner_user_id: Optional[str] = None,
|
||||
) -> Optional[Project]:
|
||||
"""Return only an explicitly owned/scoped project; legacy records fail closed."""
|
||||
if not isinstance(organization_id, str) or not organization_id:
|
||||
return None
|
||||
project = cls.get_project(project_id)
|
||||
if project is None or project.organization_id != organization_id:
|
||||
return None
|
||||
if owner_user_id is not None and project.owner_user_id != owner_user_id:
|
||||
return None
|
||||
return project
|
||||
|
||||
@classmethod
|
||||
def delete_project(cls, project_id: str) -> bool:
|
||||
def delete_project(
|
||||
cls,
|
||||
project_id: str,
|
||||
*,
|
||||
organization_id: str,
|
||||
owner_user_id: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
删除项目及其所有文件
|
||||
|
||||
删除项目及其所有文件，但只允许删除明确授权范围内的项目。
|
||||
|
||||
Args:
|
||||
project_id: 项目ID
|
||||
|
||||
organization_id: 当前请求的组织范围
|
||||
owner_user_id: 普通用户的所有者范围；管理员可留空
|
||||
|
||||
Returns:
|
||||
是否删除成功
|
||||
"""
|
||||
project_dir = cls._get_project_dir(project_id)
|
||||
|
||||
project = cls.get_project_for_scope(
|
||||
project_id,
|
||||
organization_id=organization_id,
|
||||
owner_user_id=owner_user_id,
|
||||
)
|
||||
if project is None:
|
||||
return False
|
||||
|
||||
project_dir = cls._get_project_dir(project.project_id)
|
||||
if not os.path.exists(project_dir):
|
||||
return False
|
||||
|
||||
|
||||
shutil.rmtree(project_dir)
|
||||
return True
|
||||
|
||||
|
||||
35
backend/app/models/rate_limit.py
Normal file
35
backend/app/models/rate_limit.py
Normal file
@@ -0,0 +1,35 @@
|
||||
"""Durable rate-limit event records."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import DateTime, Index, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from ..db import Base
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class RateLimitEvent(Base):
|
||||
"""One recorded rate-limit hit for an operation + key (no secrets)."""
|
||||
|
||||
__tablename__ = "rate_limit_events"
|
||||
__table_args__ = (
|
||||
Index("ix_rate_limit_op_key_created", "operation", "key", "created_at"),
|
||||
Index("ix_rate_limit_org_created", "organization_id", "created_at"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(64), primary_key=True, default=lambda: f"rl_{uuid4().hex}"
|
||||
)
|
||||
operation: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||
key: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
organization_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now()
|
||||
)
|
||||
138
backend/app/models/saas.py
Normal file
138
backend/app/models/saas.py
Normal file
@@ -0,0 +1,138 @@
|
||||
"""SQLAlchemy identity and tenant metadata models."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import List
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import CheckConstraint, DateTime, Enum as SAEnum, ForeignKey, Integer, String, UniqueConstraint, func, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from ..db import Base
|
||||
from ..security.policy import Role
|
||||
|
||||
|
||||
def _id(prefix: str) -> str:
|
||||
return f"{prefix}_{uuid4().hex}"
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _role_values(enum_type):
|
||||
return [member.value for member in enum_type]
|
||||
|
||||
|
||||
class Organization(Base):
|
||||
__tablename__ = "organizations"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: _id("org"))
|
||||
name: Mapped[str] = mapped_column(String(160), nullable=False)
|
||||
slug: Mapped[str] = mapped_column(String(80), nullable=False, unique=True, index=True)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="active", server_default=text("'active'")
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=_utc_now, server_default=func.now(), nullable=False
|
||||
)
|
||||
|
||||
memberships: Mapped[List["Membership"]] = relationship(
|
||||
back_populates="organization", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: _id("usr"))
|
||||
email_normalized: Mapped[str] = mapped_column(String(320), nullable=False, unique=True, index=True)
|
||||
password_hash: Mapped[str] = mapped_column(
|
||||
String(512), nullable=False, default="!invite_pending", server_default=text("'!invite_pending'")
|
||||
)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="active", server_default=text("'active'")
|
||||
)
|
||||
auth_version: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0, server_default=text("0")
|
||||
)
|
||||
locale: Mapped[str] = mapped_column(
|
||||
String(8), nullable=False, default="th", server_default=text("'th'")
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=_utc_now, server_default=func.now(), nullable=False
|
||||
)
|
||||
last_login_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
memberships: Mapped[List["Membership"]] = relationship(
|
||||
back_populates="user", cascade="all, delete-orphan"
|
||||
)
|
||||
sessions: Mapped[List["AuthSession"]] = relationship(
|
||||
back_populates="user", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
|
||||
class Membership(Base):
|
||||
__tablename__ = "memberships"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "organization_id", name="uq_membership_user_org"),
|
||||
CheckConstraint(
|
||||
"role IN ('super_admin', 'admin', 'user')",
|
||||
name="ck_membership_role",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: _id("mem"))
|
||||
user_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
organization_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
role: Mapped[Role] = mapped_column(
|
||||
SAEnum(
|
||||
Role,
|
||||
name="role",
|
||||
values_callable=_role_values,
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
validate_strings=True,
|
||||
),
|
||||
nullable=False,
|
||||
)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="active", server_default=text("'active'")
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=_utc_now, server_default=func.now(), nullable=False
|
||||
)
|
||||
|
||||
user: Mapped[User] = relationship(back_populates="memberships")
|
||||
organization: Mapped[Organization] = relationship(back_populates="memberships")
|
||||
sessions: Mapped[List["AuthSession"]] = relationship(back_populates="membership")
|
||||
|
||||
|
||||
class AuthSession(Base):
|
||||
__tablename__ = "sessions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: _id("ses"))
|
||||
user_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
membership_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("memberships.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
token_hash: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True)
|
||||
auth_version: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0, server_default=text("0")
|
||||
)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=_utc_now, server_default=func.now(), nullable=False
|
||||
)
|
||||
last_seen_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
user: Mapped[User] = relationship(back_populates="sessions")
|
||||
membership: Mapped[Membership] = relationship(back_populates="sessions")
|
||||
42
backend/app/models/settings.py
Normal file
42
backend/app/models/settings.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""Durable, versioned platform settings (LLM provider etc.) with redacted secrets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, ForeignKey, JSON, String, func, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from ..db import Base
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class PlatformSettings(Base):
|
||||
"""A versioned snapshot of platform LLM settings.
|
||||
|
||||
Public/non-secret settings live in ``settings`` (JSON). The API key must be
|
||||
stored encrypted (as ``secret_ref``), never as plaintext in ``settings``.
|
||||
``active`` marks the current effective version.
|
||||
"""
|
||||
|
||||
__tablename__ = "platform_settings"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(64), primary_key=True, default=lambda: f"ps_{uuid4().hex}"
|
||||
)
|
||||
version: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
||||
settings: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
secret_ref: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
updated_by_user_id: Mapped[str | None] = mapped_column(
|
||||
String(64), nullable=True
|
||||
)
|
||||
active: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default=text("0")
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now()
|
||||
)
|
||||
@@ -1,43 +1,48 @@
|
||||
"""
|
||||
任务状态管理
|
||||
用于跟踪长时间运行的任务（如图谱构建）
|
||||
"""Durable task status management with a test-only in-memory fallback.
|
||||
|
||||
The Flask application configures a SQLAlchemy session factory at startup. Code
|
||||
that uses TaskManager outside an application (small unit tests and legacy
|
||||
adapters) keeps the old in-memory behavior, but production requests do not.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Dict, Any, Optional
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, Optional, cast
|
||||
|
||||
from flask import current_app, has_app_context
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from ..models.operations import Job, JobStatus
|
||||
from ..utils.locale import t
|
||||
|
||||
|
||||
class TaskStatus(str, Enum):
|
||||
"""任务状态枚举"""
|
||||
PENDING = "pending" # 等待中
|
||||
PROCESSING = "processing" # 处理中
|
||||
COMPLETED = "completed" # 已完成
|
||||
FAILED = "failed" # 失败
|
||||
PENDING = "pending"
|
||||
PROCESSING = "processing"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Task:
|
||||
"""任务数据类"""
|
||||
task_id: str
|
||||
task_type: str
|
||||
status: TaskStatus
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
progress: int = 0 # 总进度百分比 0-100
|
||||
message: str = "" # 状态消息
|
||||
result: Optional[Dict] = None # 任务结果
|
||||
error: Optional[str] = None # 错误信息
|
||||
metadata: Dict = field(default_factory=dict) # 额外元数据
|
||||
progress_detail: Dict = field(default_factory=dict) # 详细进度信息
|
||||
|
||||
progress: int = 0
|
||||
message: str = ""
|
||||
result: Optional[Dict] = None
|
||||
error: Optional[str] = None
|
||||
metadata: Dict = field(default_factory=dict)
|
||||
progress_detail: Dict = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""转换为字典"""
|
||||
return {
|
||||
"task_id": self.task_id,
|
||||
"task_type": self.task_type,
|
||||
@@ -54,57 +59,182 @@ class Task:
|
||||
|
||||
|
||||
class TaskManager:
|
||||
"""
|
||||
任务管理器
|
||||
线程安全的任务状态管理
|
||||
"""
|
||||
|
||||
_instance = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
def __new__(cls):
|
||||
"""单例模式"""
|
||||
if cls._instance is None:
|
||||
with cls._lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
cls._instance._tasks: Dict[str, Task] = {}
|
||||
cls._instance._task_lock = threading.Lock()
|
||||
return cls._instance
|
||||
|
||||
"""Thread-safe task facade bound to one app/session factory."""
|
||||
|
||||
_configured_session_factory = None
|
||||
_config_lock = threading.Lock()
|
||||
_fallback_tasks: Dict[str, Task] = {}
|
||||
_fallback_lock = threading.Lock()
|
||||
|
||||
def __init__(self, session_factory=None):
|
||||
"""Bind this manager to an explicit or current-app session factory.
|
||||
|
||||
A manager created during a request/app context keeps that app's factory
|
||||
for background work, but it cannot be reused inside a different Flask
|
||||
app. The explicit class configuration remains only for legacy tests and
|
||||
callers that run outside Flask.
|
||||
"""
|
||||
self._bound_app = None
|
||||
if has_app_context():
|
||||
app = cast(Any, current_app)._get_current_object()
|
||||
current_factory = app.extensions.get("crowdsight_session_factory")
|
||||
if not callable(current_factory):
|
||||
raise RuntimeError("task_session_factory_required")
|
||||
if session_factory is not None and session_factory is not current_factory:
|
||||
raise RuntimeError("task_session_factory_mismatch")
|
||||
bound_factory = current_factory
|
||||
self._bound_app = app
|
||||
elif session_factory is not None:
|
||||
bound_factory = session_factory
|
||||
else:
|
||||
# No Flask app context: use the explicit legacy/test binding.
|
||||
bound_factory = type(self)._configured_session_factory
|
||||
|
||||
self._session_factory = bound_factory
|
||||
self._tasks = type(self)._fallback_tasks
|
||||
self._task_lock = type(self)._fallback_lock
|
||||
|
||||
@classmethod
|
||||
def configure(cls, session_factory) -> None:
|
||||
"""Set an explicit outside-Flask binding for tests/legacy adapters."""
|
||||
with cls._config_lock:
|
||||
cls._configured_session_factory = session_factory
|
||||
with cls._fallback_lock:
|
||||
cls._fallback_tasks.clear()
|
||||
|
||||
def _factory(self):
|
||||
if not has_app_context():
|
||||
return self._session_factory
|
||||
|
||||
app = cast(Any, current_app)._get_current_object()
|
||||
current_factory = app.extensions.get("crowdsight_session_factory")
|
||||
if not callable(current_factory):
|
||||
raise RuntimeError("task_session_factory_required")
|
||||
if self._bound_app is not None and self._bound_app is not app:
|
||||
raise RuntimeError("task_app_context_mismatch")
|
||||
if self._session_factory is not None and self._session_factory is not current_factory:
|
||||
raise RuntimeError("task_session_factory_mismatch")
|
||||
if self._session_factory is None:
|
||||
self._session_factory = current_factory
|
||||
self._bound_app = app
|
||||
return self._session_factory
|
||||
|
||||
@staticmethod
|
||||
def _bounded_text(value: Any, limit: int = 4000) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
return str(value)[:limit]
|
||||
|
||||
@staticmethod
|
||||
def _job_status(status: TaskStatus | str | None) -> str | None:
|
||||
if status is None:
|
||||
return None
|
||||
value = status.value if isinstance(status, TaskStatus) else str(status)
|
||||
return {
|
||||
TaskStatus.PENDING.value: JobStatus.QUEUED.value,
|
||||
TaskStatus.PROCESSING.value: JobStatus.RUNNING.value,
|
||||
TaskStatus.COMPLETED.value: JobStatus.SUCCEEDED.value,
|
||||
TaskStatus.FAILED.value: JobStatus.FAILED.value,
|
||||
JobStatus.QUEUED.value: JobStatus.QUEUED.value,
|
||||
JobStatus.RUNNING.value: JobStatus.RUNNING.value,
|
||||
JobStatus.SUCCEEDED.value: JobStatus.SUCCEEDED.value,
|
||||
JobStatus.FAILED.value: JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value: JobStatus.CANCELLED.value,
|
||||
}.get(value)
|
||||
|
||||
@staticmethod
|
||||
def _task_status(status: str | JobStatus) -> TaskStatus:
|
||||
value = status.value if isinstance(status, JobStatus) else str(status)
|
||||
return {
|
||||
JobStatus.QUEUED.value: TaskStatus.PENDING,
|
||||
JobStatus.RUNNING.value: TaskStatus.PROCESSING,
|
||||
JobStatus.SUCCEEDED.value: TaskStatus.COMPLETED,
|
||||
JobStatus.FAILED.value: TaskStatus.FAILED,
|
||||
JobStatus.CANCELLED.value: TaskStatus.FAILED,
|
||||
}.get(value, TaskStatus.FAILED)
|
||||
|
||||
@classmethod
|
||||
def _from_job(cls, job: Job) -> Task:
|
||||
metadata = job.job_metadata if isinstance(job.job_metadata, dict) else {}
|
||||
result = job.result if isinstance(job.result, dict) else job.result
|
||||
detail = job.progress_detail if isinstance(job.progress_detail, dict) else {}
|
||||
return Task(
|
||||
task_id=job.id,
|
||||
task_type=job.operation,
|
||||
status=cls._task_status(job.status),
|
||||
created_at=job.created_at,
|
||||
updated_at=job.updated_at,
|
||||
progress=job.progress,
|
||||
message=job.message,
|
||||
result=result,
|
||||
error=job.error_code,
|
||||
metadata=metadata,
|
||||
progress_detail=detail,
|
||||
)
|
||||
|
||||
def create_task(self, task_type: str, metadata: Optional[Dict] = None) -> str:
|
||||
"""
|
||||
创建新任务
|
||||
|
||||
Args:
|
||||
task_type: 任务类型
|
||||
metadata: 额外元数据
|
||||
|
||||
Returns:
|
||||
任务ID
|
||||
"""
|
||||
metadata = metadata or {}
|
||||
factory = self._factory()
|
||||
if factory is not None:
|
||||
organization_id = metadata.get("organization_id")
|
||||
if not isinstance(organization_id, str) or not organization_id:
|
||||
raise ValueError("task_scope_required")
|
||||
with factory() as session:
|
||||
job = Job(
|
||||
organization_id=organization_id,
|
||||
owner_user_id=metadata.get("owner_user_id"),
|
||||
project_id=metadata.get("project_id"),
|
||||
graph_id=metadata.get("graph_id"),
|
||||
operation=self._bounded_text(task_type, 120),
|
||||
status=JobStatus.QUEUED.value,
|
||||
job_metadata=metadata,
|
||||
progress_detail={},
|
||||
)
|
||||
session.add(job)
|
||||
session.commit()
|
||||
return job.id
|
||||
|
||||
task_id = str(uuid.uuid4())
|
||||
now = datetime.now()
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
task = Task(
|
||||
task_id=task_id,
|
||||
task_type=task_type,
|
||||
status=TaskStatus.PENDING,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
metadata=metadata or {}
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
with self._task_lock:
|
||||
self._tasks[task_id] = task
|
||||
|
||||
return task_id
|
||||
|
||||
def get_task(self, task_id: str) -> Optional[Task]:
|
||||
"""获取任务"""
|
||||
|
||||
def get_task(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
organization_id: Optional[str] = None,
|
||||
owner_user_id: Optional[str] = None,
|
||||
) -> Optional[Task]:
|
||||
factory = self._factory()
|
||||
if factory is not None:
|
||||
with factory() as session:
|
||||
statement = select(Job).where(Job.id == task_id)
|
||||
if organization_id is not None:
|
||||
statement = statement.where(Job.organization_id == organization_id)
|
||||
if owner_user_id is not None:
|
||||
statement = statement.where(Job.owner_user_id == owner_user_id)
|
||||
job = session.scalar(statement)
|
||||
return self._from_job(job) if job is not None else None
|
||||
with self._task_lock:
|
||||
return self._tasks.get(task_id)
|
||||
|
||||
task = self._tasks.get(task_id)
|
||||
if task is None:
|
||||
return None
|
||||
if organization_id is not None and task.metadata.get("organization_id") != organization_id:
|
||||
return None
|
||||
if owner_user_id is not None and task.metadata.get("owner_user_id") != owner_user_id:
|
||||
return None
|
||||
return task
|
||||
|
||||
def update_task(
|
||||
self,
|
||||
task_id: str,
|
||||
@@ -113,28 +243,45 @@ class TaskManager:
|
||||
message: Optional[str] = None,
|
||||
result: Optional[Dict] = None,
|
||||
error: Optional[str] = None,
|
||||
progress_detail: Optional[Dict] = None
|
||||
progress_detail: Optional[Dict] = None,
|
||||
):
|
||||
"""
|
||||
更新任务状态
|
||||
|
||||
Args:
|
||||
task_id: 任务ID
|
||||
status: 新状态
|
||||
progress: 进度
|
||||
message: 消息
|
||||
result: 结果
|
||||
error: 错误信息
|
||||
progress_detail: 详细进度信息
|
||||
"""
|
||||
factory = self._factory()
|
||||
if factory is not None:
|
||||
with factory() as session:
|
||||
job = session.get(Job, task_id)
|
||||
if job is None:
|
||||
return
|
||||
mapped_status = self._job_status(status)
|
||||
if mapped_status is not None:
|
||||
job.status = mapped_status
|
||||
if mapped_status in {
|
||||
JobStatus.SUCCEEDED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
}:
|
||||
job.finished_at = datetime.now(timezone.utc)
|
||||
if progress is not None:
|
||||
job.progress = min(max(int(progress), 0), 100)
|
||||
if message is not None:
|
||||
job.message = self._bounded_text(message)
|
||||
if result is not None:
|
||||
job.result = result
|
||||
if error is not None:
|
||||
job.error_code = self._bounded_text(error, 120)
|
||||
if progress_detail is not None:
|
||||
job.progress_detail = progress_detail
|
||||
job.updated_at = datetime.now(timezone.utc)
|
||||
session.commit()
|
||||
return
|
||||
|
||||
with self._task_lock:
|
||||
task = self._tasks.get(task_id)
|
||||
if task:
|
||||
task.updated_at = datetime.now()
|
||||
task.updated_at = datetime.now(timezone.utc)
|
||||
if status is not None:
|
||||
task.status = status
|
||||
if progress is not None:
|
||||
task.progress = progress
|
||||
task.progress = min(max(int(progress), 0), 100)
|
||||
if message is not None:
|
||||
task.message = message
|
||||
if result is not None:
|
||||
@@ -143,44 +290,71 @@ class TaskManager:
|
||||
task.error = error
|
||||
if progress_detail is not None:
|
||||
task.progress_detail = progress_detail
|
||||
|
||||
|
||||
def complete_task(self, task_id: str, result: Dict):
|
||||
"""标记任务完成"""
|
||||
self.update_task(
|
||||
task_id,
|
||||
status=TaskStatus.COMPLETED,
|
||||
progress=100,
|
||||
message=t('progress.taskComplete'),
|
||||
result=result
|
||||
message=t("progress.taskComplete"),
|
||||
result=result,
|
||||
)
|
||||
|
||||
|
||||
def fail_task(self, task_id: str, error: str):
|
||||
"""标记任务失败"""
|
||||
self.update_task(
|
||||
task_id,
|
||||
status=TaskStatus.FAILED,
|
||||
message=t('progress.taskFailed'),
|
||||
error=error
|
||||
message=t("progress.taskFailed"),
|
||||
error=error,
|
||||
)
|
||||
|
||||
def list_tasks(self, task_type: Optional[str] = None) -> list:
|
||||
"""列出任务"""
|
||||
|
||||
def list_tasks(
|
||||
self,
|
||||
task_type: Optional[str] = None,
|
||||
*,
|
||||
organization_id: Optional[str] = None,
|
||||
owner_user_id: Optional[str] = None,
|
||||
) -> list:
|
||||
factory = self._factory()
|
||||
if factory is not None:
|
||||
with factory() as session:
|
||||
statement = select(Job).order_by(Job.created_at.desc())
|
||||
if task_type:
|
||||
statement = statement.where(Job.operation == task_type)
|
||||
if organization_id is not None:
|
||||
statement = statement.where(Job.organization_id == organization_id)
|
||||
if owner_user_id is not None:
|
||||
statement = statement.where(Job.owner_user_id == owner_user_id)
|
||||
jobs = session.scalars(statement.limit(100)).all()
|
||||
return [self._from_job(job) for job in jobs]
|
||||
with self._task_lock:
|
||||
tasks = list(self._tasks.values())
|
||||
if task_type:
|
||||
tasks = [t for t in tasks if t.task_type == task_type]
|
||||
return [t.to_dict() for t in sorted(tasks, key=lambda x: x.created_at, reverse=True)]
|
||||
|
||||
tasks = [task for task in tasks if task.task_type == task_type]
|
||||
if organization_id is not None:
|
||||
tasks = [task for task in tasks if task.metadata.get("organization_id") == organization_id]
|
||||
if owner_user_id is not None:
|
||||
tasks = [task for task in tasks if task.metadata.get("owner_user_id") == owner_user_id]
|
||||
return [task for task in sorted(tasks, key=lambda item: item.created_at, reverse=True)]
|
||||
|
||||
def cleanup_old_tasks(self, max_age_hours: int = 24):
|
||||
"""清理旧任务"""
|
||||
from datetime import timedelta
|
||||
cutoff = datetime.now() - timedelta(hours=max_age_hours)
|
||||
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(hours=max_age_hours)
|
||||
factory = self._factory()
|
||||
if factory is not None:
|
||||
with factory() as session:
|
||||
session.execute(
|
||||
delete(Job).where(
|
||||
Job.created_at < cutoff,
|
||||
Job.status.in_([JobStatus.SUCCEEDED.value, JobStatus.FAILED.value]),
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return
|
||||
with self._task_lock:
|
||||
old_ids = [
|
||||
tid for tid, task in self._tasks.items()
|
||||
task_id
|
||||
for task_id, task in self._tasks.items()
|
||||
if task.created_at < cutoff and task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED]
|
||||
]
|
||||
for tid in old_ids:
|
||||
del self._tasks[tid]
|
||||
|
||||
for task_id in old_ids:
|
||||
del self._tasks[task_id]
|
||||
|
||||
45
backend/app/models/usage.py
Normal file
45
backend/app/models/usage.py
Normal file
@@ -0,0 +1,45 @@
|
||||
"""Durable LLM usage/cost events."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import DateTime, Float, ForeignKey, Index, Integer, String, func, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from ..db import Base
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class UsageEvent(Base):
|
||||
"""One LLM usage record. Never stores prompt content or secrets."""
|
||||
|
||||
__tablename__ = "usage_events"
|
||||
__table_args__ = (
|
||||
Index("ix_usage_org_created", "organization_id", "created_at"),
|
||||
Index("ix_usage_org_user", "organization_id", "user_id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(64), primary_key=True, default=lambda: f"usage_{uuid4().hex}"
|
||||
)
|
||||
organization_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
operation: Mapped[str] = mapped_column(String(160), nullable=False)
|
||||
model: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
input_tokens: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
|
||||
output_tokens: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
|
||||
estimated_cost: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.0, server_default=text("0")
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now()
|
||||
)
|
||||
Reference in New Issue
Block a user