From 8b84378fe10acde648ad9b9b21d349f27d84ce55 Mon Sep 17 00:00:00 2001 From: Kunthawat Greethong Date: Mon, 31 Aug 2026 13:05:21 +0700 Subject: [PATCH] feat: SaaS foundation for CrowdSight Elevate MiroFish/CrowdSight from single-container dev to a SaaS foundation: - Local memory backend (Zep-compatible): memory services/models, local graph builder + updater, AgentActivity seam, import-boundary isolation; Zep stays default, local is opt-in behind MEMORY_BACKEND. Semantic parity not yet proven. - Durable product persistence: projects/simulations/reports schema (migration 0007) + tenant/owner-scoped ProductRepository + dual-write + scoped_project read-first + ArtifactStore abstraction; durable JobQueue + worker.py. - SaaS hardening: durable RateLimiter (wired to login), UsageService (LLM accounting), redacted AuditService, idempotency, CORS allowlist, safe API errors, single-use PasswordResetService + endpoints (covers invite-pending). - Exactly 3 roles (super_admin/admin/user) with tenant authz policy. - Admin UI: GET/POST/PATCH /api/admin/users + GET/PUT /api/admin/settings (super-admin only, encrypted/masked); AdminView.vue + SettingsView.vue with admin/super-admin route guards, th/en i18n. - Production deploy topology: multi-stage Dockerfile (frontend build + gunicorn wsgi + nginx SPA-proxy + supervisord worker), backend/wsgi.py, gunicorn dep. Backend 197 passed; frontend 10 tests + build green. ruff unavailable (gap). No commit of credentials; secrets handled via env/.env.example. Deferred: Zep semantic A/B parity, object storage cutover, mobile QA, EasyPanel container build of deploy topology. --- .env.example | 21 +- ...26-08-23_110451-mirofish-saas-migration.md | 494 ++++ Dockerfile | 65 +- backend/alembic.ini | 38 + backend/app/__init__.py | 56 +- backend/app/api/admin.py | 222 ++ backend/app/api/agent_group.py | 14 +- backend/app/api/auth.py | 264 +++ backend/app/api/graph.py | 301 ++- backend/app/api/report.py | 374 +-- backend/app/api/simulation.py | 572 +++-- backend/app/api/template.py | 11 +- backend/app/config.py | 26 +- backend/app/db.py | 50 + backend/app/models/__init__.py | 17 +- backend/app/models/memory.py | 141 ++ backend/app/models/operations.py | 128 + backend/app/models/password_reset.py | 37 + backend/app/models/product.py | 186 ++ backend/app/models/project.py | 154 +- backend/app/models/rate_limit.py | 35 + backend/app/models/saas.py | 138 ++ backend/app/models/settings.py | 42 + backend/app/models/task.py | 366 ++- backend/app/models/usage.py | 45 + backend/app/security/__init__.py | 19 + backend/app/security/auth.py | 108 + backend/app/security/policy.py | 121 + backend/app/security/resources.py | 223 ++ backend/app/services/__init__.py | 126 +- backend/app/services/artifact_store.py | 87 + backend/app/services/audit_service.py | 63 + backend/app/services/graph_builder.py | 47 +- backend/app/services/idempotency.py | 244 ++ backend/app/services/identity.py | 260 +++ backend/app/services/job_queue.py | 82 + backend/app/services/local_graph_builder.py | 197 ++ .../services/local_graph_memory_updater.py | 177 ++ backend/app/services/memory_activity.py | 184 ++ backend/app/services/memory_entity_reader.py | 242 ++ backend/app/services/memory_extraction.py | 143 ++ backend/app/services/memory_repository.py | 310 +++ backend/app/services/memory_service.py | 122 + backend/app/services/memory_tools.py | 506 ++++ .../app/services/oasis_profile_generator.py | 88 +- backend/app/services/password_reset.py | 76 + backend/app/services/product_repository.py | 379 +++ backend/app/services/rate_limiter.py | 49 + backend/app/services/report_agent.py | 63 +- backend/app/services/settings_service.py | 115 + .../services/simulation_config_generator.py | 7 +- backend/app/services/simulation_manager.py | 134 +- backend/app/services/simulation_runner.py | 75 +- backend/app/services/usage_service.py | 69 + .../app/services/zep_graph_memory_updater.py | 177 +- backend/app/services/zep_tools.py | 16 +- backend/app/utils/api_errors.py | 43 + backend/app/utils/language_policy.py | 82 + backend/app/utils/locale.py | 31 +- backend/migrations/env.py | 58 + backend/migrations/script.py.mako | 26 + backend/migrations/versions/0001_identity.py | 79 + backend/migrations/versions/0002_sessions.py | 45 + backend/migrations/versions/0003_memory.py | 116 + .../migrations/versions/0004_operations.py | 107 + .../migrations/versions/0005_job_payload.py | 22 + .../migrations/versions/0006_job_metadata.py | 18 + .../versions/0007_product_resources.py | 119 + .../versions/0008_platform_settings.py | 28 + .../migrations/versions/0009_rate_limit.py | 41 + .../migrations/versions/0010_usage_events.py | 39 + .../versions/0011_password_reset_tokens.py | 30 + backend/pyproject.toml | 11 +- .../memory_parity/entity_reader_fixture.json | 49 + .../fixtures/memory_parity/tools_fixture.json | 75 + backend/tests/test_admin_users_api.py | 235 ++ backend/tests/test_api_errors.py | 48 + .../test_api_no_raw_exception_details.py | 36 + backend/tests/test_artifact_store.py | 70 + backend/tests/test_audit_service.py | 127 + backend/tests/test_auth_api.py | 81 + backend/tests/test_auth_service.py | 87 + backend/tests/test_authorization_policy.py | 126 + backend/tests/test_auxiliary_api_security.py | 249 ++ backend/tests/test_child_resource_scope.py | 41 + backend/tests/test_cors_config.py | 43 + backend/tests/test_durable_task_manager.py | 209 ++ backend/tests/test_graph_auth_scope.py | 101 + backend/tests/test_graph_backend_routes.py | 201 ++ .../tests/test_graph_builder_task_scope.py | 49 + backend/tests/test_idempotency_api.py | 184 ++ backend/tests/test_job_queue.py | 161 ++ backend/tests/test_language_policy.py | 47 + backend/tests/test_local_golden_flow.py | 239 ++ backend/tests/test_local_graph_builder.py | 134 ++ .../tests/test_local_graph_memory_updater.py | 50 + backend/tests/test_local_reader_factory.py | 31 + backend/tests/test_local_report_tools.py | 62 + backend/tests/test_local_service_import.py | 168 ++ backend/tests/test_local_zep_boundary.py | 34 + backend/tests/test_locale_runtime.py | 40 + backend/tests/test_memory_backend_switch.py | 131 ++ backend/tests/test_memory_entity_reader.py | 59 + backend/tests/test_memory_extraction.py | 92 + backend/tests/test_memory_parity.py | 154 ++ backend/tests/test_memory_repository.py | 76 + backend/tests/test_memory_service.py | 88 + backend/tests/test_memory_tools.py | 39 + backend/tests/test_password_reset.py | 186 ++ backend/tests/test_product_persistence.py | 781 +++++++ backend/tests/test_profile_memory_mode.py | 60 + backend/tests/test_project_store_security.py | 33 + backend/tests/test_project_tenant_scope.py | 68 + backend/tests/test_rate_limiter.py | 172 ++ backend/tests/test_report_api_contract.py | 44 + backend/tests/test_resource_auth_scope.py | 174 ++ backend/tests/test_saas_operations_schema.py | 259 +++ backend/tests/test_saas_schema.py | 84 + backend/tests/test_settings_service.py | 117 + .../tests/test_simulation_memory_injection.py | 27 + backend/tests/test_usage_service.py | 120 + backend/tests/test_worker.py | 86 + backend/uv.lock | 182 ++ backend/worker.py | 110 + backend/wsgi.py | 13 + docs/HANDOFF.md | 91 + docs/engineering-log.md | 32 + .../2026-08-23-architecture-study.md | 42 + .../2026-08-23-local-memory-first-consumer.md | 28 + .../2026-08-23-local-memory-foundation.md | 32 + ...2026-08-23-saas-foundation-continuation.md | 46 + .../2026-08-24-graph-scope-review.md | 35 + ...26-08-24-memory-parity-panorama-insight.md | 40 + ...2026-08-24-saas-foundation-verification.md | 46 + .../2026-08-24-simulation-memory-fallback.md | 53 + docs/test-evidence/2026-08-23-baseline.md | 27 + .../2026-08-23-saas-foundation.md | 50 + ...2026-08-24-saas-foundation-verification.md | 42 + frontend/index.html | 19 +- frontend/src/App.vue | 2 +- frontend/src/api/error.js | 15 + frontend/src/api/index.js | 54 +- frontend/src/api/report.js | 11 +- frontend/src/components/HistoryDatabase.vue | 2 +- frontend/src/components/Step1GraphBuild.vue | 6 +- frontend/src/components/Step2EnvSetup.vue | 12 +- frontend/src/components/Step3Simulation.vue | 6 +- frontend/src/components/Step4Report.vue | 138 +- frontend/src/components/Step5Interaction.vue | 4 +- frontend/src/i18n/index.js | 31 +- frontend/src/router/index.js | 79 +- frontend/src/stores/auth.js | 54 + frontend/src/views/AdminView.vue | 161 ++ frontend/src/views/LoginView.vue | 168 ++ frontend/src/views/Process.vue | 2068 ----------------- frontend/src/views/SettingsView.vue | 153 ++ frontend/src/views/SimulationView.vue | 2 +- frontend/tests/api-error.test.mjs | 43 + frontend/tests/auth-contract.test.mjs | 22 + frontend/tests/i18n-contract.test.mjs | 71 + frontend/tests/report-api-contract.test.mjs | 18 + locales/en.json | 65 +- locales/languages.json | 26 +- locales/th.json | 65 +- locales/zh.json | 693 +----- 165 files changed, 15884 insertions(+), 4001 deletions(-) create mode 100644 .hermes/plans/2026-08-23_110451-mirofish-saas-migration.md create mode 100644 backend/alembic.ini create mode 100644 backend/app/api/admin.py create mode 100644 backend/app/api/auth.py create mode 100644 backend/app/db.py create mode 100644 backend/app/models/memory.py create mode 100644 backend/app/models/operations.py create mode 100644 backend/app/models/password_reset.py create mode 100644 backend/app/models/product.py create mode 100644 backend/app/models/rate_limit.py create mode 100644 backend/app/models/saas.py create mode 100644 backend/app/models/settings.py create mode 100644 backend/app/models/usage.py create mode 100644 backend/app/security/__init__.py create mode 100644 backend/app/security/auth.py create mode 100644 backend/app/security/policy.py create mode 100644 backend/app/security/resources.py create mode 100644 backend/app/services/artifact_store.py create mode 100644 backend/app/services/audit_service.py create mode 100644 backend/app/services/idempotency.py create mode 100644 backend/app/services/identity.py create mode 100644 backend/app/services/job_queue.py create mode 100644 backend/app/services/local_graph_builder.py create mode 100644 backend/app/services/local_graph_memory_updater.py create mode 100644 backend/app/services/memory_activity.py create mode 100644 backend/app/services/memory_entity_reader.py create mode 100644 backend/app/services/memory_extraction.py create mode 100644 backend/app/services/memory_repository.py create mode 100644 backend/app/services/memory_service.py create mode 100644 backend/app/services/memory_tools.py create mode 100644 backend/app/services/password_reset.py create mode 100644 backend/app/services/product_repository.py create mode 100644 backend/app/services/rate_limiter.py create mode 100644 backend/app/services/settings_service.py create mode 100644 backend/app/services/usage_service.py create mode 100644 backend/app/utils/api_errors.py create mode 100644 backend/app/utils/language_policy.py create mode 100644 backend/migrations/env.py create mode 100644 backend/migrations/script.py.mako create mode 100644 backend/migrations/versions/0001_identity.py create mode 100644 backend/migrations/versions/0002_sessions.py create mode 100644 backend/migrations/versions/0003_memory.py create mode 100644 backend/migrations/versions/0004_operations.py create mode 100644 backend/migrations/versions/0005_job_payload.py create mode 100644 backend/migrations/versions/0006_job_metadata.py create mode 100644 backend/migrations/versions/0007_product_resources.py create mode 100644 backend/migrations/versions/0008_platform_settings.py create mode 100644 backend/migrations/versions/0009_rate_limit.py create mode 100644 backend/migrations/versions/0010_usage_events.py create mode 100644 backend/migrations/versions/0011_password_reset_tokens.py create mode 100644 backend/tests/fixtures/memory_parity/entity_reader_fixture.json create mode 100644 backend/tests/fixtures/memory_parity/tools_fixture.json create mode 100644 backend/tests/test_admin_users_api.py create mode 100644 backend/tests/test_api_errors.py create mode 100644 backend/tests/test_api_no_raw_exception_details.py create mode 100644 backend/tests/test_artifact_store.py create mode 100644 backend/tests/test_audit_service.py create mode 100644 backend/tests/test_auth_api.py create mode 100644 backend/tests/test_auth_service.py create mode 100644 backend/tests/test_authorization_policy.py create mode 100644 backend/tests/test_auxiliary_api_security.py create mode 100644 backend/tests/test_child_resource_scope.py create mode 100644 backend/tests/test_cors_config.py create mode 100644 backend/tests/test_durable_task_manager.py create mode 100644 backend/tests/test_graph_auth_scope.py create mode 100644 backend/tests/test_graph_backend_routes.py create mode 100644 backend/tests/test_graph_builder_task_scope.py create mode 100644 backend/tests/test_idempotency_api.py create mode 100644 backend/tests/test_job_queue.py create mode 100644 backend/tests/test_language_policy.py create mode 100644 backend/tests/test_local_golden_flow.py create mode 100644 backend/tests/test_local_graph_builder.py create mode 100644 backend/tests/test_local_graph_memory_updater.py create mode 100644 backend/tests/test_local_reader_factory.py create mode 100644 backend/tests/test_local_report_tools.py create mode 100644 backend/tests/test_local_service_import.py create mode 100644 backend/tests/test_local_zep_boundary.py create mode 100644 backend/tests/test_locale_runtime.py create mode 100644 backend/tests/test_memory_backend_switch.py create mode 100644 backend/tests/test_memory_entity_reader.py create mode 100644 backend/tests/test_memory_extraction.py create mode 100644 backend/tests/test_memory_parity.py create mode 100644 backend/tests/test_memory_repository.py create mode 100644 backend/tests/test_memory_service.py create mode 100644 backend/tests/test_memory_tools.py create mode 100644 backend/tests/test_password_reset.py create mode 100644 backend/tests/test_product_persistence.py create mode 100644 backend/tests/test_profile_memory_mode.py create mode 100644 backend/tests/test_project_store_security.py create mode 100644 backend/tests/test_project_tenant_scope.py create mode 100644 backend/tests/test_rate_limiter.py create mode 100644 backend/tests/test_report_api_contract.py create mode 100644 backend/tests/test_resource_auth_scope.py create mode 100644 backend/tests/test_saas_operations_schema.py create mode 100644 backend/tests/test_saas_schema.py create mode 100644 backend/tests/test_settings_service.py create mode 100644 backend/tests/test_simulation_memory_injection.py create mode 100644 backend/tests/test_usage_service.py create mode 100644 backend/tests/test_worker.py create mode 100644 backend/worker.py create mode 100644 backend/wsgi.py create mode 100644 docs/HANDOFF.md create mode 100644 docs/engineering-log.md create mode 100644 docs/engineering-log/2026-08-23-architecture-study.md create mode 100644 docs/engineering-log/2026-08-23-local-memory-first-consumer.md create mode 100644 docs/engineering-log/2026-08-23-local-memory-foundation.md create mode 100644 docs/engineering-log/2026-08-23-saas-foundation-continuation.md create mode 100644 docs/engineering-log/2026-08-24-graph-scope-review.md create mode 100644 docs/engineering-log/2026-08-24-memory-parity-panorama-insight.md create mode 100644 docs/engineering-log/2026-08-24-saas-foundation-verification.md create mode 100644 docs/engineering-log/2026-08-24-simulation-memory-fallback.md create mode 100644 docs/test-evidence/2026-08-23-baseline.md create mode 100644 docs/test-evidence/2026-08-23-saas-foundation.md create mode 100644 docs/test-evidence/2026-08-24-saas-foundation-verification.md create mode 100644 frontend/src/api/error.js create mode 100644 frontend/src/stores/auth.js create mode 100644 frontend/src/views/AdminView.vue create mode 100644 frontend/src/views/LoginView.vue delete mode 100644 frontend/src/views/Process.vue create mode 100644 frontend/src/views/SettingsView.vue create mode 100644 frontend/tests/api-error.test.mjs create mode 100644 frontend/tests/auth-contract.test.mjs create mode 100644 frontend/tests/i18n-contract.test.mjs create mode 100644 frontend/tests/report-api-contract.test.mjs diff --git a/.env.example b/.env.example index c62780a..5fcd598 100644 --- a/.env.example +++ b/.env.example @@ -1,8 +1,14 @@ # ================================================================ -# CrowdSight 环境变量配置 +# CrowdSight environment configuration # ================================================================ -# 复制此文件为 .env 并填入你的 API 密钥： +# Copy this file to .env and fill in deployment secrets: # cp .env.example .env +# +# Security (generate a unique value for every deployment) +SECRET_KEY=replace_with_a_long_random_secret +SESSION_COOKIE_SECURE=true +CORS_ALLOWED_ORIGINS=https://your-frontend.example.com + # # LLM 配置支持两种方式： # 方式1（推荐）：设置 LLM_PROVIDER，只需提供 API Key @@ -61,11 +67,12 @@ LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 LLM_MODEL_NAME=qwen-plus -# ================================================================ -# Zep 记忆图谱配置（必需） -# ================================================================ -# 每月免费额度即可支撑简单使用 -# 获取地址: https://app.getzep.com/ +# Memory backend migration switch: +# zep = legacy Zep runtime (current default) +# local = local SQLAlchemy memory repository (requires imported memory graphs) +MEMORY_BACKEND=zep + +# Legacy Zep credential; remove only after all consumers are cut over. ZEP_API_KEY=your_zep_api_key_here diff --git a/.hermes/plans/2026-08-23_110451-mirofish-saas-migration.md b/.hermes/plans/2026-08-23_110451-mirofish-saas-migration.md new file mode 100644 index 0000000..611a463 --- /dev/null +++ b/.hermes/plans/2026-08-23_110451-mirofish-saas-migration.md @@ -0,0 +1,494 @@ +# MiroFish → Company SaaS Architecture & Migration Plan + +**สถานะ:** Implementation in progress — bounded SaaS foundation and local-memory E2E gates are implemented; production migration remains incomplete +**Repo:** `/Users/kunthawat/Gitea/MiroFish` +**HEAD ที่ตรวจ:** `89d04e7` (`fix: replace hero logo with inline use cases grid`) +**วันที่ตรวจ:** 2026-08-23 (+07:00) + +## 1. เป้าหมาย + +เปลี่ยน MiroFish/CrowdSight จากแอป single-user ที่พึ่งพา Zep Cloud และไฟล์ในเครื่อง ให้เป็น SaaS ของบริษัท โดยมีผลลัพธ์ที่ยอมรับได้ดังนี้: + +1. **Frontend ไม่มีภาษาจีนใน product surface** — UI chrome, locale fallback, API error ที่แสดงบนจอ, prompt/runtime ที่ระบบสร้าง, route metadata และ build artifact ใช้เฉพาะ Thai หรือ English +2. **ไม่มี Zep dependency/runtime** — ใช้ LLM ทำ extraction/merge/summarization/reasoning และใช้ local durable graph repository ทำ storage/index/query แทน +3. **มี authentication + tenant isolation + 3 roles เท่านั้น** — `super_admin`, `admin`, `user` +4. **Super admin ตั้งค่าระบบเชิงลึกได้** — provider, base URL, model, generation/runtime parameters และ policy ที่เกี่ยวข้อง โดยไม่เปิด secret ให้ browser +5. **Admin ใช้งานระบบและจัดการ users ในองค์กรของตนได้** — ห้ามเปลี่ยน/มอบ `super_admin` +6. **User ใช้งาน simulation/report ของตนได้เท่านั้น** +7. **ระบบรองรับการ deploy แบบ SaaS จริง** — ไม่พึ่ง in-memory state หรือ background thread ภายใน web process เป็นหลัก + +## 2. สิ่งที่ตรวจพบจาก baseline + +### 2.1 Baseline ที่ผ่าน/ไม่ผ่าน + +- `git status --short --branch`: clean, branch `main` ตรงกับ `origin/main` +- `npm run build`: **ผ่าน**; Vite build สำเร็จ แต่เตือน chunk หลักเกิน 500 kB และ dynamic/static import ของ `pendingUpload.js` +- `python3 -m compileall -q backend/app backend/run.py backend/scripts`: **ผ่าน** (syntax-only; เครื่องปัจจุบันเป็น Python 3.14 แต่โปรเจกต์กำหนด `<3.13` จึงยังไม่ใช่ runtime validation) +- `git diff --check`: **ผ่าน** +- ไม่พบ test suite จริงที่ครอบคลุม backend/frontend; พบเพียง `backend/scripts/test_profile_format.py` +- Backend มี **64 routes** ใน 5 blueprints และยังไม่มี authentication middleware +- Frontend มี 16 `.vue` files; source frontend 22 files มี CJK code points รวมจำนวนมาก และ build artifact ยังมี CJK อยู่ จึง **ยังไม่ผ่าน** acceptance ของข้อ 1 + +### 2.2 ปัญหาหลักที่ยืนยันจาก source + +| พื้นที่ | หลักฐาน | ผลกระทบ | +|---|---|---| +| i18n default/fallback | `frontend/src/i18n/index.js:17,22` ตั้ง default/fallback เป็น `zh` | ผู้ใช้ใหม่/ผู้ใช้เก่าที่มี `localStorage.locale=zh` อาจกลับไป Chinese | +| HTML metadata/font | `frontend/index.html:2,4,7,11-12` ใช้ `lang=zh`, default `zh`, `Noto Sans SC`, title/description Chinese | browser chrome/metadata และ build ยังเผย Chinese | +| locale registry | `locales/languages.json:2-4` มี `zh` และ instruction ให้ตอบ Chinese | LLM output และ language switcher ยังมี Chinese path | +| backend locale fallback | `backend/app/utils/locale.py:30-32,36-49,66-69` fallback เป็น `zh` | API/background task อาจสร้างข้อความ Chinese แม้ frontend เลือกภาษาอื่น | +| hard-coded frontend UI | `frontend/src/views/Process.vue` มี rendered Chinese หลายจุด; `frontend/src/components/Step4Report.vue:1349+` มี parser ที่ผูกกับ Chinese headings | แค่แก้ locale JSON ไม่พอ และ parser จะพังเมื่อ output เป็น Thai/English | +| hard-coded frontend prompt | `frontend/src/components/Step5Interaction.vue:731-733` สร้าง prompt ด้วย Chinese labels | ระบบส่ง Chinese เข้า LLM จาก frontend runtime | +| old Zep terminology | `frontend/src/views/Process.vue:320`, `locales/en.json`/`th.json` มี Zep strings | product UI ยังสื่อว่าต้องมี Zep | +| Zep package/config | `backend/pyproject.toml:19-20`, `backend/requirements.txt:16-17`, `backend/uv.lock` | dependency และ lockfile ยังบังคับ Zep | +| Zep graph build | `backend/app/services/graph_builder.py:13-18,121-180,205-291,294-505` | ontology, batch episodes, async processing, temporal graph info ผูกกับ SDK | +| Zep read/search | `backend/app/services/zep_entity_reader.py:10,127-180,215-331`; `backend/app/services/zep_tools.py:425-544,650-1090,1145-1270` | profile generation/report agent ต้องการ node/edge/search/temporal result | +| Zep runtime updates | `backend/app/services/zep_graph_memory_updater.py:15,202-246,396-455` | ทุก activity ถูกแปลงเป็น text แล้วส่งเข้า Zep | +| ReportAgent coupling | `backend/app/services/report_agent.py:25-30,883-907,1156-1178` | report tools รับ `ZepToolsService` โดยตรง | +| ไม่มี auth | `backend/app/__init__.py:65-73`; route เช่น `backend/app/api/graph.py:36-67` | ทุกคนที่เข้าถึง API รู้ project ID ก็อ่าน/แก้/ลบ resource ได้ | +| CORS กว้างเกินไป | `backend/app/__init__.py:42-43` ใช้ `origins: "*"` | ไม่เหมาะกับ authenticated SaaS | +| insecure defaults | `backend/app/config.py:65-69` มี default secret และ DEBUG=True | เสี่ยง production และ session/auth ในอนาคต | +| single-user storage | `backend/app/models/project.py:101-219` ใช้ directory เดียวทั้งระบบ | ไม่มี owner/org scope, query, transaction หรือ multi-instance safety | +| ephemeral state | `backend/app/models/task.py:56-72`; `backend/app/services/simulation_runner.py:219-224` | restart/หลาย worker ทำให้ task/process state หายหรือแยกกัน | +| production runtime | `Dockerfile:18-29` ติดตั้งและรัน `npm run dev`; `docker-compose.yml:1-14` มี service เดียว/volume | ยังไม่ใช่ web/API/worker/data topology สำหรับ SaaS | +| raw request logging | `backend/app/__init__.py:51-57` log JSON request body | อาจบันทึก source document, simulation requirement, chat history หรือ secret ลง log ต้อง redaction/ปิดใน production | +| raw exception/path exposure | หลาย route คืน `str(e)`; resource paths ต่อจาก caller id เช่น `project.py:113-120`, `simulation_manager.py:139-143`, `report_agent.py:1910-1918` | อาจเปิด traceback, filesystem path หรือข้อมูลภายใน และเสี่ยง path traversal ต้องใช้ scoped repository + safe opaque IDs + generic error envelope | +| retry/idempotency | `frontend/src/api/index.js:64-76` retry POST บางประเภท เช่น graph/simulation/report | response หลุดหลัง server ทำงานสำเร็จอาจสร้าง project/job/LLM cost ซ้ำ ต้องมี idempotency key และ durable job deduplication | +| contract drift | `frontend/src/api/report.js:15-16` เรียก status ด้วย GET/query แต่ backend `report.py:203-230` รับ POST/body | ต้องทำ API contract tests ก่อนเพิ่ม role-specific UI | +| browser-only draft state | `frontend/src/store/pendingUpload.js:7-31` เก็บ pending upload ใน memory | reload/tab close ทำให้ไฟล์และ intent หาย; SaaS ควรใช้ server-side draft หรือ presigned upload session | +| branding/deployment mismatch | `package.json:2-4`, `README.md:3-9`, `docker-compose.yml:3` ใช้ CrowdSight ขณะที่ repo/task ใช้ MiroFish | ต้องตัดสินใจชื่อ product/canonical identifiers ก่อนทำ SaaS auth, domains และเอกสาร | + +## 3. หลักการตัดสินใจ + +### 3.1 ข้อเสนอที่ควรยึด + +- **อย่าใช้ LLM แทน database/search engine โดยตรง**: LLM ทำ extraction, entity resolution, summary, query decomposition และ reranking ได้ แต่ไม่ควรรับผิดชอบ durable storage, pagination, exact ID lookup หรือ authorization +- **ใช้ PostgreSQL เป็น system of record** ตั้งแต่ต้นสำหรับ SaaS; ใช้ SQLite ได้เฉพาะ local test/dev หากต้องการ +- **ออกแบบ tenant boundary ตั้งแต่วันแรก** แม้เริ่มจากบริษัทเดียว เพื่อไม่ต้องรื้อ schema ภายหลัง +- **เก็บ internal enum/field names เป็น English stable identifiers**; label ที่ user เห็นให้มาจาก `en/th` locale +- **LLM settings ต้อง snapshot ตอนเริ่ม job** เพื่อให้ report/simulation reproducible แม้ super admin เปลี่ยน model ระหว่างรัน +- **Frontend ไม่ควร parse Markdown ที่ผูกกับภาษา**; API ควรส่ง structured result แล้วให้ frontend render ด้วย locale +- **ห้ามส่ง raw exception ให้ผู้ใช้**; log รายละเอียดไว้ server-side และส่ง `error_code` + localized message + +### 3.2 Non-goals ของ v1 + +- Billing/subscription/usage metering +- SSO/SAML/SCIM +- Public self-signup แบบเปิดกว้าง +- สิทธิ์แบบ custom role นอกเหนือจาก 3 role +- การรับประกันว่า LLM ให้ผล semantic เหมือน Zep 100% — จะทำ parity ที่ interface/behavior และวัดด้วย golden fixtures แทน + +## 4. Target architecture + +```text +Browser (Vue SPA) + ├─ Login / Auth store / route guards + ├─ User workspace + ├─ Admin user management + └─ Super-admin settings + │ HTTPS, HttpOnly auth cookie, Accept-Language: th|en + ▼ +Flask API (stateless web process) + ├─ auth + role + tenant policy + ├─ project/simulation/report API + ├─ memory API adapter + ├─ settings/audit API + └─ job enqueue/status API + │ + ├──────── PostgreSQL ──────── users, orgs, projects, graph, jobs, reports, settings, audit + ├──────── Redis/queue ─────── graph extraction, report generation, simulation orchestration + └──────── Object storage ──── uploads, reports, simulation artifacts/logs + +Worker(s) + ├─ Memory extraction/merge worker (LLM) + ├─ Report worker (LLM + memory tools) + └─ OASIS simulation worker + +LLM Provider Gateway + ├─ provider/base_url/model selected by effective settings + ├─ server-side secret resolution + ├─ retry/timeout/rate/cost policy + └─ structured output validation +``` + +## 5. Domain/data model ที่เสนอ + +### 5.1 Identity/tenant + +- `organizations`: `id`, `name`, `slug`, `status`, timestamps +- `users`: `id`, `email_normalized`, `password_hash` or external identity subject, `status`, `auth_version`, `locale`, `last_login_at`, timestamps +- `memberships`: `user_id`, `organization_id`, `role` (`super_admin|admin|user`), status, timestamps; unique `(user_id, organization_id)` +- `sessions` หรือ `refresh_tokens`: hashed token, user, expiry, revoked_at, rotation metadata + +ใช้ memberships แทนการผูก user กับองค์กรเดียวแบบถาวร เพื่อรองรับ super-admin/platform scope และการเพิ่มหลายองค์กรภายหลัง โดยยังคงมี role เพียง 3 ค่า +- `audit_logs`: actor, organization, action, target type/id, metadata, timestamp, IP/user-agent ที่จำเป็น + +### 5.2 Product resources + +- `projects`: `organization_id`, `owner_user_id`, name/status, source metadata, ontology JSON, language, timestamps +- `simulations`: `organization_id`, `project_id`, `created_by`, status, config snapshot, worker/job id, timestamps +- `reports`: `organization_id`, `project_id`, `simulation_id`, created_by, status, outline/sections/content metadata, timestamps +- `jobs`: durable status/progress/error/result reference, `idempotency_key`, retry count, usage/cost metadata; ห้ามพึ่ง in-memory `TaskManager` +- `artifacts`: object key, checksum, content type, size, owner resource, retention metadata +- `usage_events`: organization/user, operation, model, input/output tokens, estimated cost, timestamps +- `audit_logs`: actor, organization, action, target and redacted metadata + +Report agent logs contain prompts, tool results and model responses; treat them as tenant data, redact secrets and expose only through scoped role-aware endpoints + +### 5.3 Local graph/memory ที่มาแทน Zep + +- `memory_graphs`: graph id, organization, project, ontology JSON, build status/version +- `memory_episodes`: graph id, source type (`document|simulation_action|manual`), source reference, raw/normalized text, processing status, timestamps +- `memory_nodes`: graph id, canonical name, normalized name, labels, attributes JSONB, summary, aliases JSONB, confidence, created/updated timestamps +- `memory_edges`: graph id, source node, target node, relation name, fact, attributes JSONB, confidence, `valid_at`, `invalid_at`, `expired_at`, created timestamp +- `memory_evidence`: episode-to-node/edge links, evidence span/reference, extractor version +- optional `memory_embeddings`: pgvector or external vector index; defer until baseline local search is measured + +Unique/index rules: + +- unique `(graph_id, normalized_canonical_name)` where applicable +- indexes on graph, labels, relation, source/target, temporal fields +- full-text/trigram index for quick search +- all repository methods require `organization_id`/graph scope; never accept an unscoped graph id from a route + +### 5.4 Settings + +- `platform_settings`: global active LLM provider/model/runtime settings, version, updated_by +- optional `organization_settings`: only if later allowing per-org overrides +- secret values are encrypted server-side or referenced from a secret manager; API returns masked status only +- job stores a settings snapshot/version, not secret plaintext + +## 6. Zep replacement design + +### 6.1 Preserve a compatibility interface, replace implementation + +Create an internal interface such as `MemoryRepository` / `MemorySearchService` whose output contracts preserve what the current UI/report code needs: + +- `NodeInfo`: uuid, name, labels, summary, attributes +- `EdgeInfo`: uuid, name, fact, source/target ids, temporal fields +- `EntityNode` / `FilteredEntities` +- `SearchResult`: facts, edges, nodes, query, total_count +- `PanoramaResult`: active/historical facts, all nodes/edges, counts +- `InsightForgeResult`: sub_queries, semantic facts, entity insights, relationship chains, counts + +Then change consumers: + +- `GraphBuilderService` → `GraphMemoryBuilder` +- `ZepEntityReader` → `MemoryEntityReader` +- `ZepToolsService` → `MemoryToolsService` +- `ZepGraphMemoryUpdater` → `MemoryEventProcessor` +- `OasisProfileGenerator` receives `MemorySearchService` +- `ReportAgent` depends on an abstract memory tools interface, not a Zep-named class + +This keeps API/frontend changes bounded while removing the external provider. + +### 6.2 LLM extraction contract + +LLM must return JSON only and pass Pydantic validation. Proposed shape: + +```json +{ + "entities": [ + { + "mention": "text span", + "canonical_name": "stable name", + "labels": ["Person"], + "aliases": [], + "attributes": {}, + "summary": "short evidence-grounded summary", + "confidence": 0.0 + } + ], + "edges": [ + { + "source_entity_ref": "entity-1", + "target_entity_ref": "entity-2", + "relation": "WORKS_FOR", + "fact": "evidence-grounded fact", + "attributes": {}, + "valid_at": null, + "invalid_at": null, + "expired_at": null, + "confidence": 0.0, + "evidence": ["episode-id or span-id"] + } + ], + "episode_summary": "short summary", + "unresolved_mentions": [] +} +``` + +Prompt requirements: + +1. System prompt is English and neutral; put the selected Thai/English output instruction at the **start**. +2. Never invent a fact not supported by the episode/context. +3. Use only ontology labels/relation names; field names and enum values are stable English identifiers. +4. Preserve temporal semantics; use `null` when dates are not evidenced. +5. Return entity references inside the same response, never database ids guessed by the model. +6. Keep evidence references for audit/debugging. +7. Return valid JSON with bounded array/string lengths. + +Server-side post-processing must: + +- normalize names/aliases +- resolve references against deterministic candidate search first, then let LLM choose among candidates or create a new node +- reject unknown labels/relations +- clamp confidence and numeric fields +- deduplicate edges +- upsert transactionally +- record extractor prompt/version/model in job metadata, never API key + +### 6.3 Entity resolution and temporal update + +1. Extract candidates from a chunk. +2. Retrieve possible existing nodes using normalized name, aliases, trigram/full-text, and optionally embeddings. +3. Ask LLM only to choose `existing_node_id` or `new_entity`, with evidence. +4. Upsert node and edges in one transaction. +5. When a new fact contradicts an active fact, mark the previous edge `invalid_at`/`expired_at`; do not delete history. +6. Rebuild summaries from canonical facts/events through a separate LLM prompt. + +### 6.4 Search parity + +- **Quick search**: local full-text/trigram retrieval of facts/nodes; optional LLM query rewrite only. +- **Panorama search**: deterministic graph scan with active/historical classification and bounded result set. +- **InsightForge**: LLM decomposes the question into subqueries; local retrieval gathers candidates; deterministic dedupe; optional LLM reranker/synthesizer can rank candidate IDs but may not create unsupported facts. +- **Entity context**: fetch node + adjacent edges + related nodes from repository. +- **Interview agents**: remains an OASIS operation, not graph search; keep separate from memory repository. + +The user-facing contract can be equivalent to current Zep-backed dataclasses, but the semantic result will only be considered acceptable after golden-fixture comparison. + +### 6.5 Dynamic simulation memory + +Do not generate Chinese natural-language episodes in the frontend/backend. Convert OASIS actions to canonical event JSON first: + +```json +{ + "simulation_id": "...", + "platform": "twitter", + "agent_id": 12, + "action_type": "CREATE_POST", + "action_args": {}, + "round_num": 4, + "timestamp": "..." +} +``` + +Use deterministic mappings for obvious actions (`FOLLOW`, `LIKE_POST`, `REPOST`) and LLM extraction only for content/stance/context enrichment. Queue batches through the worker and persist failure/retry status. + +### 6.6 Migration decision + +- **Recommended if no production Zep data exists:** clean break; rebuild graph from stored source documents under the new repository. +- **If existing Zep data matters:** before removing credentials, run an export/import job for nodes, edges, temporal fields, episodes and evidence; verify counts/checksums and sample search behavior. Keep import code as a one-time script, not runtime dependency. + +## 7. Prompt architecture + +Centralize prompts in a versioned module, for example `backend/app/services/prompts/`: + +- `ontology.py` +- `memory_extraction.py` +- `entity_resolution.py` +- `memory_summary.py` +- `query_decomposition.py` +- `report.py` +- `profile.py` + +Every prompt receives an explicit `OutputLanguage` (`th` or `en`) and a `prompt_version`. Use structured output validators and retry with a repair prompt that contains the validation error, not an unbounded second generation. + +Important current gaps to fix during this work: + +- `backend/app/services/ontology_generator.py:284-309` still builds the user message with Chinese headings. +- `backend/app/services/zep_tools.py:1138-1143` has Chinese fallback subqueries. +- several config/profile/report prompts append language instructions inconsistently; move to one prompt builder that prepends a strong instruction. +- API error handlers currently sometimes return `str(e)`; use stable codes/localized messages instead. + +## 8. SaaS authorization model + +### 8.1 Role matrix + +| Capability | `super_admin` | `admin` | `user` | +|---|---:|---:|---:| +| Login/use simulation | yes | yes | yes | +| View own projects/reports | yes | yes | yes | +| View all projects in own org | yes | yes | no (recommended) | +| Manage users in own org | yes | yes | no | +| Assign `user` role | yes | yes | no | +| Grant/revoke `admin` | yes | **no, recommended** | no | +| Grant/revoke `super_admin` | yes | no | no | +| Platform/org settings | yes | no | no | +| LLM provider/model/base URL | yes | no | no | +| Audit logs | all/platform scope | own org scope | no | + +The backend is authoritative. Hiding a menu in Vue is not authorization. + +### 8.2 Auth recommendation + +- Login with email + password. +- Argon2id (or Werkzeug scrypt) password hash; never store raw password. +- Short-lived access session in `HttpOnly`, `Secure`, `SameSite` cookie; refresh/session rotation server-side. +- CSRF protection for cookie-authenticated state-changing requests. +- Rate-limit login, password reset and admin user mutations. +- Normalize email and enforce unique `(organization_id, email_normalized)`. +- On role/status/password change, revoke sessions through `auth_version` or token revocation. +- Do not store auth tokens in `localStorage`. + +### 8.3 Resource policy + +Every project/simulation/report/graph/job route must: + +1. authenticate request +2. load resource through repository with tenant scope +3. apply role policy +4. only then read/write files or start a worker + +Never trust `project_id`, `simulation_id` or `graph_id` supplied by the browser as proof of ownership. + +### 8.4 API surface + +Add: + +- `POST /api/auth/login` +- `POST /api/auth/logout` +- `GET /api/auth/me` +- `POST /api/auth/refresh` or session refresh +- `POST /api/auth/change-password` +- `GET/POST /api/users` — admin/super admin; scoped +- `GET/PATCH/DELETE /api/users/` — policy-enforced; soft delete/deactivate +- `POST /api/users//reset-password` or invite flow +- `GET/PATCH /api/admin/settings/llm` — super admin only +- `POST /api/admin/settings/llm/test` — super admin only, redacted response +- `GET /api/admin/audit-logs` — scoped by role + +Protect all existing graph/simulation/report/template/agent-group routes. Keep `/health` unauthenticated but do not expose config/secrets. + +## 9. Frontend migration + +1. Add auth store and `/login`; add route guard and role-aware navigation. +2. Add `/admin/users` for admin/super admin. +3. Add `/admin/settings` for super admin only. +4. Keep user workspace flow but scope API calls to authenticated identity; never put role authority only in Vue. +5. Limit supported locales to `th` and `en`; recommend default `th`, with English switcher. +6. Normalize legacy `localStorage.locale=zh` to `th`/`en` during boot and overwrite it. +7. Remove CJK from `frontend/index.html`, `App.vue` font stack, hard-coded templates/strings, regex/parser labels, prompts and generated build. +8. Replace language-specific report/tool parsing with structured API fields. For old Chinese report artifacts, either migrate/translate before serving or mark them as legacy and do not display unprocessed content if the strict no-Chinese requirement applies to historical data too. +9. Rename/remove unused Chinese-named assets and repository docs as a separate cleanup pass; do not assume an image filename is harmless if it is later imported into the SPA. + +Acceptance gate for frontend: + +```text +- no `zh` locale, `zh-CN`, `Noto Sans SC`, Chinese language label, or Chinese fallback in shipped frontend +- no CJK code point in user-facing frontend source/locale/meta/build artifact, excluding explicitly approved user-uploaded content fixtures +- all 16 Vue views/components render with `th` and `en` +- API errors shown on screen are localized `th`/`en` +- browser storage containing legacy `zh` self-heals to an allowed locale +``` + +## 10. Settings design for super admin + +Safe editable fields: + +- provider preset and display name +- model name +- base URL allowlist/custom endpoint policy +- temperature, max tokens, timeout, retry count/backoff +- ontology/memory extraction batch size and token budget +- report max tool calls/reflection rounds/token budget +- OASIS max rounds/concurrency/retention limits +- default UI/output language (`th|en`) +- feature flags for memory search modes + +Guardrails: + +- server validates ranges and URL scheme/allowlist +- API key field is write-only/masked; never return it +- save creates version + audit log +- test connection uses the pending settings without persisting unless explicitly saved +- each job captures effective settings version/model/base URL (not secret) +- admin/user cannot mutate these settings + +## 11. Implementation phases and gates + +### M0 — Decision lock and contracts + +- Confirm language scope, tenancy, Zep data migration, onboarding, canonical product branding and deployment topology. +- Freeze API/error/graph schemas, role constants and HTTP method/body contracts. +- Define idempotency semantics for every LLM-triggering POST before enabling frontend retries. +- Preserve the current clean baseline. +- Record and fix known frontend/backend contract drift, including report status method/body mismatch, before adding route guards. + +**Gate:** decisions recorded; no implementation starts against unresolved identity/data assumptions. + +### M1 — Persistence, auth and tenant foundation + +- Add migrations and PostgreSQL repository. +- Add org/user/session/audit/job/resource tables. +- Add auth endpoints, password hashing or managed identity integration, cookies/CSRF, role decorators/policies. +- Attach `organization_id` and `owner_user_id` to product resources. +- Replace unscoped file manager calls with scoped repositories and explicit path confinement. +- Add idempotency keys, per-user/org rate limits, concurrency limits, input/output caps and LLM usage/cost accounting. +- Redact request bodies, raw exceptions, filesystem paths, tracebacks and sensitive report logs from user-facing responses/logs. +- Add contract tests for method/body mismatches and child-resource relationship checks. + +**Gate:** automated role × endpoint × cross-tenant matrix passes; IDOR attempts return 404/403 without leakage. + +### M2 — No-Zep memory repository + +- Add graph/episode/node/edge/evidence tables and repository. +- Implement structured LLM extraction, validation, merge, temporal update and summaries. +- Implement quick/panorama/insight/entity-context adapters. +- Replace profile/report/updater dependencies. +- Remove Zep package, env var, imports and runtime checks. + +**Gate:** no `zep_cloud` import/dependency/config; golden fixtures compare node/edge counts, entity recall, search recall@k, temporal classification and failure recovery against captured baseline or approved acceptance thresholds. + +### M3 — Frontend Thai/English hardening + +- Remove `zh` registry/fallback/default and legacy browser state. +- Translate hard-coded rendered strings and backend error codes. +- Replace report parser with structured JSON contract. +- Remove Zep product terminology and CJK from shipped frontend. +- Add login/role guards and admin navigation shell. + +**Gate:** build + CJK scanner + visual/manual smoke at desktop and 320×568 / 500×768; both locales complete all core journeys. + +### M4 — Admin and super-admin surfaces + +- User list/create/invite/deactivate/reset/role policy. +- Super-admin LLM settings, test connection, versioning and audit view. +- Redacted settings API and permission tests. + +**Gate:** each forbidden control is blocked server-side and hidden/disabled client-side; audit entries exist for sensitive changes. + +### M5 — Worker/deployment hardening + +- Move graph/report/simulation work to durable jobs/worker processes. +- Decide Redis/queue and object storage; make artifact paths tenant-scoped. +- Replace process-local pending uploads with server-side draft/upload sessions. +- Replace dev Docker command with production frontend/API/worker services. +- Add rate limits, structured redacted logs, metrics, usage/cost metering, retention and backup/restore procedure. + +**Gate:** restart web process during a job does not lose job state; two workers do not cross tenant/resource boundaries; deploy/rollback smoke passes. + +### M6 — Migration and release verification + +- Rebuild from source or import Zep data according to M0 decision. +- Run security scan, tests, E2E, locale scan, worker smoke, backup/restore and cost/latency benchmark. +- Document rollback and known limitations. + +- Project deletion has an explicit, tested cascade/retention policy for project files, graphs, simulations, reports, jobs and artifacts. +- The frontend/backend API contract has no untested method/body drift; all retryable mutations are idempotent. +- Request and report logs are redacted and cannot expose prompts, API keys, tracebacks or filesystem paths to unauthorized roles. + +**Gate:** release checklist has evidence, not just green intentions. + +## 12. Questions/decisions required before implementation + +1. **ภาษาใน source content:** ต้องการห้าม Chinese เฉพาะ UI/system-generated text หรือแม้แต่ชื่อ entity, post, report ที่มาจากเอกสารที่ผู้ใช้อัปโหลดด้วย? — แนะนำ: UI และ generated system text ต้องไม่มี Chinese; user-provided source data ให้เก็บ original แต่มี display translation/locale policy แยก +2. **Tenant scope:** ต้องการหลายบริษัท/หลายองค์กรตั้งแต่ v1 หรือบริษัทเดียวก่อน? — แนะนำ: ทำ `organization_id` ตั้งแต่ v1 แม้เปิดใช้บริษัทเดียว +3. **Admin onboarding:** ใช้ email invite/SES หรือให้ admin สร้าง account โดยตรง? — แนะนำ: email invite + one-time setup token; ห้ามส่ง password ถาวรผ่านแชต/อีเมล +4. **Zep data:** มี graph/project ที่ต้องรักษาไว้หรือ rebuild ได้? — แนะนำ: ถ้ายังไม่มี production data ให้ clean rebuild; ถ้ามี ให้ export/import ก่อนถอด Zep +5. **LLM settings scope:** global ทั้งแพลตฟอร์มหรือแยกต่อองค์กร? — แนะนำ: global ใน v1, แต่ schema รองรับ org override ภายหลัง +6. **Deployment:** ยอมรับ PostgreSQL + Redis + object storage/volume แยกหรือไม่? — แนะนำ: production SaaS ต้องแยก; single-container เป็นแค่ local/demo + +## 13. Immediate next action after approval + +ทำ M0 ให้จบด้วยคำตอบ 6 ข้อด้านบน แล้วแตก implementation plan เป็น PR-sized batches โดยเริ่มจาก **M1 persistence/auth contract** และ **M3 locale contract** ก่อนแตะ memory replacement; ห้ามเริ่มจากการลบ Zep imports แบบกระจาย เพราะจะทำให้ไม่มี storage/search contract รองรับและเสี่ยงทำ behavior เดิมหาย. diff --git a/Dockerfile b/Dockerfile index e656468..170e073 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,62 @@ -FROM python:3.11 +# ============================================================ +# CrowdSight production image (multi-service, EasyPanel-buildable) +# +# Services inside one container (supervisord): +# - web: nginx serving the built SPA, proxying /api -> gunicorn :5001 +# - backend: gunicorn WSGI (wsgi:app) on 0.0.0.0:5001 +# - worker: durable PollingWorker (backend/worker.py) +# ============================================================ + +# ---- Stage 1: build the frontend SPA ---- +FROM node:20 AS frontend-build +WORKDIR /build +COPY package.json package-lock.json* ./ +COPY frontend/package.json frontend/package-lock.json* ./frontend/ +# install root + frontend deps +RUN npm ci 2>/dev/null || true; npm ci --prefix frontend || true +COPY locales ./locales +COPY frontend ./frontend +RUN npm run build --prefix frontend + +# ---- Stage 2: runtime (python + nginx + supervisord) ---- +FROM python:3.11-slim AS runtime + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH=/app/backend -# 安装 Node.js （满足 >=18）及必要工具 RUN apt-get update \ - && apt-get install -y --no-install-recommends nodejs npm \ + && apt-get install -y --no-install-recommends nginx supervisor \ && rm -rf /var/lib/apt/lists/* -# 从 uv 官方镜像复制 uv +# uv runtime COPY --from=ghcr.io/astral-sh/uv:0.9.26 /uv /uvx /bin/ WORKDIR /app -# 先复制依赖描述文件以利用缓存 -COPY package.json package-lock.json ./ -COPY frontend/package.json frontend/package-lock.json ./frontend/ +# Install backend deps first (cache-friendly) COPY backend/pyproject.toml backend/uv.lock ./backend/ +RUN cd backend && uv sync --frozen --no-dev -# 安装依赖（Node + Python） -RUN npm ci \ - && npm ci --prefix frontend \ - && cd backend && uv sync --frozen +# Copy project source +COPY backend ./backend +COPY locales ./locales +COPY package.json ./ -# 复制项目源码 -COPY . . +# Copy built SPA into nginx web root +COPY --from=frontend-build /build/frontend/dist /usr/share/nginx/html -EXPOSE 3000 5001 +# nginx config: SPA + /api proxy to gunicorn +RUN echo 'server {\n listen 8080;\n server_name _;\n root /usr/share/nginx/html;\n index index.html;\n location / { try_files $uri $uri/ /index.html; }\n location /api/ {\n proxy_pass http://127.0.0.1:5001;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header X-Forwarded-Proto $scheme;\n }\n}\n' > /etc/nginx/sites-available/crowdsight \ + && ln -sf /etc/nginx/sites-available/crowdsight /etc/nginx/sites-enabled/crowdsight \ + && rm -f /etc/nginx/sites-enabled/default -# 同时启动前后端（开发模式） -CMD ["npm", "run", "dev"] \ No newline at end of file +# supervisor: run nginx + gunicorn + worker +RUN echo '[supervisord]\nnodeamon=false\n\n[program:nginx]\ncommand=/usr/sbin/nginx -g "daemon off;"\nautostart=true\nautorestart=true\n\n[program:backend]\ncommand=/bin/bash -c "cd /app/backend && uv run gunicorn -w 2 -b 0.0.0.0:5001 --timeout 120 wsgi:app"\nautostart=true\nautorestart=true\n\n[program:worker]\ncommand=/bin/bash -c "cd /app/backend && uv run python worker.py --poll-interval 5"\nautostart=true\nautorestart=true\n' > /etc/supervisor/conf.d/crowdsight.conf + +EXPOSE 8080 5001 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:5001/health', timeout=4)" || exit 1 + +CMD ["/usr/bin/supervisord", "-n"] diff --git a/backend/alembic.ini b/backend/alembic.ini new file mode 100644 index 0000000..92eeb9b --- /dev/null +++ b/backend/alembic.ini @@ -0,0 +1,38 @@ +[alembic] +script_location = %(here)s/migrations +prepend_sys_path = . +sqlalchemy.url = + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/backend/app/__init__.py b/backend/app/__init__.py index fc2e880..9410ca5 100644 --- a/backend/app/__init__.py +++ b/backend/app/__init__.py @@ -9,10 +9,14 @@ import warnings # 需要在所有其他导入之前设置 warnings.filterwarnings("ignore", message=".*resource_tracker.*") -from flask import Flask, request +from flask import Flask, jsonify, request from flask_cors import CORS +from werkzeug.exceptions import HTTPException from .config import Config +from .db import create_database_engine, create_session_factory +from .utils.api_errors import ApiError, internal_error_payload +from .utils.locale import t from .utils.logger import setup_logger, get_logger @@ -20,16 +24,42 @@ def create_app(config_class=Config): """Flask应用工厂函数""" app = Flask(__name__) app.config.from_object(config_class) - - # 设置JSON编码：确保中文直接显示（而不是 \uXXXX 格式） + if "*" in app.config.get("CORS_ALLOWED_ORIGINS", []): + raise RuntimeError("wildcard_cors_not_allowed") + + database_engine = create_database_engine(os.environ.get("DATABASE_URL")) + app.extensions["crowdsight_database_engine"] = database_engine + app.extensions["crowdsight_session_factory"] = create_session_factory(database_engine) + + # JSON config: keep Unicode characters readable in API responses. # Flask >= 2.3 使用 app.json.ensure_ascii，旧版本使用 JSON_AS_ASCII 配置 if hasattr(app, 'json') and hasattr(app.json, 'ensure_ascii'): app.json.ensure_ascii = False - # 设置日志 + # Configure server-side logging before registering error handlers. logger = setup_logger('crowdsight') - - # 只在 reloader 子进程中打印启动信息（避免 debug 模式下打印两次） + + @app.errorhandler(ApiError) + def handle_api_error(error: ApiError): + return jsonify(error.to_payload(t)), error.status_code + + @app.errorhandler(HTTPException) + def handle_http_error(error: HTTPException): + logger.warning("HTTP request failed: status=%s", error.code) + api_error = ApiError( + code=f"http_{error.code or 500}", + status_code=error.code or 500, + message_key="api.requestError", + ) + return jsonify(api_error.to_payload(t)), error.code or 500 + + @app.errorhandler(Exception) + def handle_unexpected_error(error: Exception): + # Keep exception details in server logs only; never serialize them. + logger.exception("Unhandled request error: %s", type(error).__name__) + return jsonify(internal_error_payload(t)), 500 + + # Only startup state is logged; request bodies are intentionally excluded. is_reloader_process = os.environ.get('WERKZEUG_RUN_MAIN') == 'true' debug_mode = app.config.get('DEBUG', False) should_log_startup = not debug_mode or is_reloader_process @@ -39,8 +69,12 @@ def create_app(config_class=Config): logger.info("CrowdSight Backend 启动中...") logger.info("=" * 50) - # 启用CORS - CORS(app, resources={r"/api/*": {"origins": "*"}}) + # Explicit allowlist only; wildcard CORS is incompatible with auth cookies. + CORS( + app, + resources={r"/api/*": {"origins": app.config.get("CORS_ALLOWED_ORIGINS", [])}}, + supports_credentials=True, + ) # 注册模拟进程清理函数（确保服务器关闭时终止所有模拟进程） from .services.simulation_runner import SimulationRunner @@ -54,7 +88,7 @@ def create_app(config_class=Config): logger = get_logger('crowdsight.request') logger.debug(f"请求: {request.method} {request.path}") if request.content_type and 'json' in request.content_type: - logger.debug(f"请求体: {request.get_json(silent=True)}") + logger.debug("JSON request received: path=%s", request.path) @app.after_request def log_response(response): @@ -64,9 +98,13 @@ def create_app(config_class=Config): # 注册蓝图 from .api import graph_bp, simulation_bp, report_bp + from .api.admin import admin_bp + from .api.auth import auth_bp from .api.template import template_bp from .api.agent_group import agent_group_bp app.register_blueprint(graph_bp, url_prefix='/api/graph') + app.register_blueprint(auth_bp, url_prefix='/api/auth') + app.register_blueprint(admin_bp, url_prefix='/api/admin') app.register_blueprint(simulation_bp, url_prefix='/api/simulation') app.register_blueprint(report_bp, url_prefix='/api/report') app.register_blueprint(template_bp, url_prefix='/api/template') diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py new file mode 100644 index 0000000..8b2619e --- /dev/null +++ b/backend/app/api/admin.py @@ -0,0 +1,222 @@ +"""Tenant-scoped user-management endpoints.""" + +from __future__ import annotations + +from flask import Blueprint, g, jsonify, request +from sqlalchemy.exc import IntegrityError + +from ..security.auth import current_actor, require_roles +from ..security.policy import ( + AuthorizationError, + Role, + assert_can_manage_llm_settings, + assert_can_manage_user, +) +from ..services.identity import IdentityRepository +from ..utils.api_errors import ApiError +from ..utils.locale import t + + +admin_bp = Blueprint("admin", __name__) + + +@admin_bp.errorhandler(ApiError) +def handle_admin_error(error: ApiError): + return jsonify(error.to_payload(t)), error.status_code + + +def _organization_for_request(repo: IdentityRepository, *, payload: dict): + context = g.auth_context + actor = current_actor() + if actor.role is not Role.SUPER_ADMIN: + return context.organization + + organization_id = payload.get("organization_id") or request.args.get("organization_id") + organization_slug = payload.get("organization_slug") or request.args.get("organization_slug") + organization = ( + repo.get_organization(organization_id) + if isinstance(organization_id, str) and organization_id + else repo.get_organization_by_slug(organization_slug) + if isinstance(organization_slug, str) and organization_slug + else None + ) + if organization is None: + raise ApiError("organization_required", 400, "api.organizationRequired") + return organization + + +def _serialize_user(user, membership): + return { + "id": user.id, + "email": user.email_normalized, + "status": user.status, + "locale": user.locale, + "role": membership.role.value, + "created_at": user.created_at.isoformat() if user.created_at else None, + } + + +@admin_bp.get("/users") +@require_roles(Role.ADMIN, Role.SUPER_ADMIN) +def list_users(): + repo = IdentityRepository(g.db_session) + organization = _organization_for_request(repo, payload={}) + rows = repo.list_users_with_memberships(organization.id) + users = [_serialize_user(user, membership) for user, membership in rows] + return jsonify({"success": True, "data": {"users": users, "count": len(users)}}) + + +@admin_bp.post("/users") +@require_roles(Role.ADMIN, Role.SUPER_ADMIN) +def create_user(): + payload = request.get_json(silent=True) or {} + if not isinstance(payload, dict): + raise ApiError("invalid_request", 400, "api.requestError") + email = payload.get("email") + if not isinstance(email, str) or not email.strip(): + raise ApiError("invalid_email", 400, "api.invalidEmail") + + requested_role = payload.get("role", Role.USER.value) + try: + role = requested_role if isinstance(requested_role, Role) else Role(requested_role) + except (TypeError, ValueError) as exc: + raise ApiError("invalid_role", 400, "api.invalidRole") from exc + + repo = IdentityRepository(g.db_session) + organization = _organization_for_request(repo, payload=payload) + actor = current_actor() + try: + assert_can_manage_user( + actor, + target_organization_id=organization.id, + target_role=role, + platform_scope=actor.role is Role.SUPER_ADMIN, + ) + except AuthorizationError as exc: + raise ApiError(exc.code, 403, "common.error") from exc + + try: + user = repo.create_user(email=email) + membership = repo.create_membership(user.id, organization.id, role) + except IntegrityError as exc: + raise ApiError("user_exists", 409, "api.userExists") from exc + except ValueError as exc: + code = str(exc) + if code == "invalid_email": + raise ApiError("invalid_email", 400, "api.invalidEmail") from exc + raise ApiError("user_creation_failed", 400, "api.requestError") from exc + + return jsonify({ + "success": True, + "data": _serialize_user(user, membership), + }), 201 + + +@admin_bp.patch("/users/") +@require_roles(Role.ADMIN, Role.SUPER_ADMIN) +def update_user(user_id: str): + """Update a user's membership role (and optionally status) in the org.""" + payload = request.get_json(silent=True) or {} + if not isinstance(payload, dict): + raise ApiError("invalid_request", 400, "api.requestError") + + repo = IdentityRepository(g.db_session) + organization = _organization_for_request(repo, payload=payload) + actor = current_actor() + + target_user = None + target_membership = None + for user, membership in repo.list_users_with_memberships(organization.id): + if user.id == user_id: + target_user = user + target_membership = membership + break + if target_user is None or target_membership is None: + raise ApiError("user_not_found", 404, "api.userNotFound") + + # Role change (if provided). + if "role" in payload: + requested_role = payload["role"] + try: + new_role = requested_role if isinstance(requested_role, Role) else Role(requested_role) + except (TypeError, ValueError) as exc: + raise ApiError("invalid_role", 400, "api.invalidRole") from exc + try: + assert_can_manage_user( + actor, + target_organization_id=organization.id, + target_role=new_role, + platform_scope=actor.role is Role.SUPER_ADMIN, + ) + except AuthorizationError as exc: + raise ApiError(exc.code, 403, "common.error") from exc + target_membership.role = new_role if isinstance(new_role, Role) else Role(new_role) + + # Status change (if provided); only super_admin may do so (safety). + if "status" in payload: + status = payload["status"] + if status not in ("active", "disabled"): + raise ApiError("invalid_status", 400, "api.invalidRequest") + if actor.role is not Role.SUPER_ADMIN: + raise ApiError("admin_status_change_forbidden", 403, "common.error") + target_user.status = status + + g.db_session.commit() + return jsonify({ + "success": True, + "data": _serialize_user(target_user, target_membership), + }) + + +def _require_platform_settings_access(): + actor = current_actor() + try: + assert_can_manage_llm_settings(actor, platform_scope=True) + except AuthorizationError as exc: + raise ApiError(exc.code, 403, "common.error") from exc + return actor + + +@admin_bp.get("/settings") +@require_roles(Role.SUPER_ADMIN) +def get_settings(): + """Return the active (masked) platform LLM settings for a super admin.""" + from ..services.settings_service import SettingsService + + _require_platform_settings_access() + svc = SettingsService(g.db_session) + return jsonify({"success": True, "data": svc.active_settings()}) + + +@admin_bp.put("/settings") +@require_roles(Role.SUPER_ADMIN) +def update_settings(): + """Persist a new version of platform LLM settings (super admin only).""" + from ..services.settings_service import SettingsService + + actor = _require_platform_settings_access() + payload = request.get_json(silent=True) or {} + if not isinstance(payload, dict): + raise ApiError("invalid_request", 400, "api.requestError") + + settings_fields = { + "provider": payload.get("provider"), + "model": payload.get("model"), + "base_url": payload.get("base_url"), + } + settings = {k: v for k, v in settings_fields.items() if isinstance(v, str) and v} + if payload.get("api_key"): + api_key = payload["api_key"] + if not isinstance(api_key, str) or not api_key.strip(): + raise ApiError("invalid_api_key", 400, "api.invalidRequest") + else: + api_key = None + + svc = SettingsService(g.db_session) + version = svc.save_settings( + settings, + api_key=api_key, + updated_by=actor.user_id, + ) + g.db_session.commit() + return jsonify({"success": True, "data": svc.active_settings()}) diff --git a/backend/app/api/agent_group.py b/backend/app/api/agent_group.py index 5e93be9..75f728f 100644 --- a/backend/app/api/agent_group.py +++ b/backend/app/api/agent_group.py @@ -9,6 +9,9 @@ from flask import Blueprint, request, jsonify from ..utils.llm_client import LLMClient from ..utils.locale import t, get_language_instruction from ..utils.logger import get_logger +from ..utils.api_errors import internal_error_payload +from ..security.auth import require_auth +from ..services.idempotency import idempotent logger = get_logger('crowdsight.agent_group') @@ -16,6 +19,8 @@ agent_group_bp = Blueprint('agent_group', __name__) @agent_group_bp.route('/categorize', methods=['POST']) +@require_auth +@idempotent def categorize_agents(): """ Categorize agents into groups based on their profiles. @@ -130,11 +135,12 @@ Categorize these agents into groups. Mark groups as default_enabled=false if the }) except Exception as e: - logger.error(f"Agent categorization failed: {e}") - return jsonify({'success': False, 'error': str(e)}), 500 + logger.error("Agent categorization failed: error_type=%s", type(e).__name__) + return jsonify(internal_error_payload(t)), 500 @agent_group_bp.route('/filter', methods=['POST']) +@require_auth def filter_agents(): """ Filter agents based on selected groups. @@ -169,5 +175,5 @@ def filter_agents(): }) except Exception as e: - logger.error(f"Agent filtering failed: {e}") - return jsonify({'success': False, 'error': str(e)}), 500 + logger.error("Agent filtering failed: error_type=%s", type(e).__name__) + return jsonify(internal_error_payload(t)), 500 diff --git a/backend/app/api/auth.py b/backend/app/api/auth.py new file mode 100644 index 0000000..8c8248b --- /dev/null +++ b/backend/app/api/auth.py @@ -0,0 +1,264 @@ +"""Authentication endpoints for the first SaaS foundation slice.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +from flask import Blueprint, current_app, g, jsonify, request + +from ..security.auth import issue_csrf_token, require_auth +from ..services.identity import IdentityRepository, PasswordService, SessionService +from ..utils.api_errors import ApiError +from ..utils.locale import t + + +auth_bp = Blueprint("auth", __name__) +SESSION_COOKIE = "crowdsight_session" + + +@auth_bp.errorhandler(ApiError) +def handle_auth_error(error: ApiError): + return jsonify(error.to_payload(t)), error.status_code + + +def _session_factory(): + factory = current_app.extensions.get("crowdsight_session_factory") + if factory is None: + raise ApiError("auth_unavailable", 503, "api.internalError") + return factory + + +# Login attempts allowed per 15-minute window per email key. +_LOGIN_LIMIT = 5 +_LOGIN_WINDOW_MINUTES = 15 + + +def _enforce_login_rate_limit(email: str) -> None: + """Reject excessive login attempts for an email (brute-force protection).""" + from datetime import timedelta + + from ..services.rate_limiter import RateLimiter + + normalized = (email or "").strip().casefold() + if not normalized: + return + factory = _session_factory() + with factory() as session: + limiter = RateLimiter( + session, window=timedelta(minutes=_LOGIN_WINDOW_MINUTES), limit=_LOGIN_LIMIT + ) + allowed = limiter.check_and_record("login", key=normalized) + if not allowed: + raise ApiError("too_many_attempts", 429, "api.tooManyAttempts") + + +def _record_audit( + *, organization_id, actor_user_id=None, action, target_type, target_id=None, details=None +): + """Best-effort audit record; never raises on audit-store failure.""" + try: + from ..services.audit_service import AuditService + + factory = _session_factory() + with factory() as session: + AuditService(session).record( + organization_id=organization_id, + actor_user_id=actor_user_id, + action=action, + target_type=target_type, + target_id=target_id, + details=details, + ) + session.commit() + except Exception: + pass + + +def _serialize_identity(context): + return { + "user": { + "id": context.user.id, + "email": context.user.email_normalized, + "locale": context.user.locale, + }, + "organization": { + "id": context.organization.id, + "name": context.organization.name, + "slug": context.organization.slug, + }, + "role": context.membership.role.value, + } + + +def _select_membership(repo, user, organization_slug: str | None): + memberships = repo.list_active_memberships(user.id) + if organization_slug: + normalized_slug = organization_slug.strip().casefold() + for membership, organization in memberships: + if organization.slug == normalized_slug: + return membership, organization + raise ApiError("organization_not_found", 404, "api.organizationNotFound") + if len(memberships) != 1: + raise ApiError("organization_required", 400, "api.organizationRequired") + return memberships[0] + + +@auth_bp.post("/login") +def login(): + data = request.get_json(silent=True) or {} + if not isinstance(data, dict): + raise ApiError("invalid_request", 400, "api.requestError") + email = data.get("email") + password = data.get("password") + if not isinstance(email, str) or not isinstance(password, str): + raise ApiError("invalid_credentials", 401, "api.invalidCredentials") + + _enforce_login_rate_limit(email) + + factory = _session_factory() + with factory() as session: + repo = IdentityRepository(session) + try: + user = repo.get_user_by_email(email) + except ValueError: + user = None + + if user is None or user.status != "active": + raise ApiError("invalid_credentials", 401, "api.invalidCredentials") + if not PasswordService.verify_password(user.password_hash, password): + raise ApiError("invalid_credentials", 401, "api.invalidCredentials") + + membership, organization = _select_membership( + repo, user, data.get("organization_slug") + ) + raw_token, _stored = SessionService.create(session, user, membership.id) + user.last_login_at = datetime.now(timezone.utc) + session.commit() + + _record_audit( + organization_id=organization.id, + actor_user_id=user.id, + action="auth.login", + target_type="user", + target_id=user.id, + details={"method": "password"}, + ) + + with factory() as session: + context = type( + "LoginContext", + (), + {"user": user, "membership": membership, "organization": organization}, + )() + response = jsonify({"success": True, "data": _serialize_identity(context)}) + response.set_cookie( + SESSION_COOKIE, + raw_token, + max_age=SessionService.DEFAULT_TTL_SECONDS, + httponly=True, + secure=bool(current_app.config.get("SESSION_COOKIE_SECURE", False)), + samesite="Lax", + ) + response.set_cookie( + "crowdsight_csrf", + issue_csrf_token(), + max_age=SessionService.DEFAULT_TTL_SECONDS, + httponly=False, + secure=bool(current_app.config.get("SESSION_COOKIE_SECURE", False)), + samesite="Lax", + ) + return response + + +@auth_bp.get("/me") +def me(): + raw_token = request.cookies.get(SESSION_COOKIE) + factory = _session_factory() + with factory() as session: + context = SessionService.resolve(session, raw_token or "") + if context is None: + raise ApiError("unauthorized", 401, "common.unauthorized") + session.commit() + return jsonify({"success": True, "data": _serialize_identity(context)}) + + +@auth_bp.post("/logout") +@require_auth +def logout(): + raw_token = request.cookies.get(SESSION_COOKIE, "") + SessionService.revoke(g.db_session, raw_token) + + response = jsonify({"success": True, "data": {"logged_out": True}}) + response.delete_cookie(SESSION_COOKIE) + response.delete_cookie("crowdsight_csrf") + return response + + +@auth_bp.post("/password-reset/request") +def password_reset_request(): + """Request a password reset for an email (always returns success).""" + from datetime import timedelta + + from ..services.password_reset import DEFAULT_TTL, PasswordResetService + + data = request.get_json(silent=True) or {} + email = data.get("email") + if not isinstance(email, str) or not email.strip(): + raise ApiError("invalid_email", 400, "api.invalidEmail") + + factory = _session_factory() + with factory() as session: + repo = IdentityRepository(session) + user = repo.get_user_by_email(email) + if user is not None: + svc = PasswordResetService(session, ttl=DEFAULT_TTL) + _token = svc.create_token(user_id=user.id) + session.commit() + # Enrolment-agnostic response avoids account-enumeration. + return jsonify({"success": True, "data": {"sent": True}}) + + +@auth_bp.post("/password-reset/confirm") +def password_reset_confirm(): + """Set a new password using a valid reset token.""" + from ..services.identity import PasswordService + from ..services.password_reset import PasswordResetService + + data = request.get_json(silent=True) or {} + token = data.get("token") + email = data.get("email") + new_password = data.get("password") + if not isinstance(token, str) or not token: + raise ApiError("invalid_token", 400, "api.invalidRequest") + if not isinstance(email, str) or not isinstance(new_password, str): + raise ApiError("invalid_credentials", 401, "api.invalidCredentials") + + factory = _session_factory() + org_id = None + with factory() as session: + repo = IdentityRepository(session) + user = repo.get_user_by_email(email) + if user is None: + raise ApiError("invalid_credentials", 401, "api.invalidCredentials") + new_hash = PasswordService.hash_password(new_password) + svc = PasswordResetService(session) + ok = svc.consume_token(token, user_id=user.id) + if not ok: + raise ApiError("invalid_token", 400, "api.invalidRequest") + user.password_hash = new_hash + session.flush() + membership = ( + repo.list_active_memberships(user.id)[0] + if repo.list_active_memberships(user.id) + else None + ) + if membership is not None: + org_id = membership[1].id + session.commit() + _record_audit( + organization_id=org_id, + actor_user_id=user.id, + action="auth.password_reset", + target_type="user", + ) + return jsonify({"success": True, "data": {"reset": True}}) diff --git a/backend/app/api/graph.py b/backend/app/api/graph.py index 3907f97..a08a761 100644 --- a/backend/app/api/graph.py +++ b/backend/app/api/graph.py @@ -4,27 +4,107 @@ """ import os -import traceback import threading -from flask import request, jsonify +from flask import current_app, request, jsonify from . import graph_bp from ..config import Config from ..services.ontology_generator import OntologyGenerator from ..services.graph_builder import GraphBuilderService +from ..services.local_graph_builder import LocalGraphBuilderService +from ..utils.llm_client import LLMClient from ..services.text_processor import TextProcessor from ..utils.file_parser import FileParser from ..utils.logger import get_logger from ..utils.locale import t, get_locale, set_locale +from ..security.auth import current_actor, require_auth +from ..security.policy import Role +from ..utils.api_errors import ApiError from ..models.task import TaskManager, TaskStatus from ..models.project import ProjectManager, ProjectStatus +from ..services.product_repository import ProductRepository +from ..services.idempotency import idempotent # 获取日志器 logger = get_logger('crowdsight.api') +@graph_bp.errorhandler(ApiError) +def handle_graph_error(error: ApiError): + return jsonify(error.to_payload(t)), error.status_code + + +def _scoped_project(project_id: str): + actor = current_actor() + owner_user_id = actor.user_id if actor.role is Role.USER else None + return ProjectManager.get_project_for_scope( + project_id, + organization_id=actor.organization_id, + owner_user_id=owner_user_id, + ) + + +def _sync_project_to_durable(project, session_factory=None): + """Best-effort dual-write of a project into the durable SQL repository. + + Never raises: if no session factory is available (non-local backend, or a + background thread without one), the filesystem manager remains the + authoritative store and the durable copy is simply skipped. Callers inside a + worker thread pass the captured ``session_factory`` explicitly. + """ + try: + if session_factory is None: + session_factory = current_app.extensions.get("crowdsight_session_factory") + if session_factory is None: + return + session = session_factory() + try: + ProductRepository(session).sync_project(project, commit=True) + finally: + session.close() + except Exception: + logger.warning("durable project sync skipped", exc_info=True) + + +def _scoped_graph(graph_id: str): + actor = current_actor() + owner_user_id = actor.user_id if actor.role is Role.USER else None + return ProjectManager.find_project_by_graph_id( + graph_id, + organization_id=actor.organization_id, + owner_user_id=owner_user_id, + ) + + +def _local_graph_builder(project): + """Create the local graph adapter with the current tenant scope.""" + session_factory = current_app.extensions.get("crowdsight_session_factory") + if not callable(session_factory): + raise ApiError("memory_backend_unavailable", 503, "api.internalError") + actor = current_actor() + return LocalGraphBuilderService( + session_factory, + organization_id=actor.organization_id, + project_id=project.project_id, + language=get_locale(), + ) + + +def _scoped_task(task_id: str): + task = TaskManager().get_task(task_id) + if task is None: + return None + actor = current_actor() + metadata = task.metadata or {} + if metadata.get("organization_id") != actor.organization_id: + return None + if actor.role is Role.USER and metadata.get("owner_user_id") != actor.user_id: + return None + return task + + def allowed_file(filename: str) -> bool: - """检查文件扩展名是否允许""" + """ตรวจสอบนามสกุลไฟล์ที่อนุญาต""" if not filename or '.' not in filename: return False ext = os.path.splitext(filename)[1].lower().lstrip('.') @@ -34,11 +114,12 @@ def allowed_file(filename: str) -> bool: # ============== 项目管理接口 ============== @graph_bp.route('/project/', methods=['GET']) +@require_auth def get_project(project_id: str): """ 获取项目详情 """ - project = ProjectManager.get_project(project_id) + project = _scoped_project(project_id) if not project: return jsonify({ @@ -53,12 +134,19 @@ def get_project(project_id: str): @graph_bp.route('/project/list', methods=['GET']) +@require_auth def list_projects(): """ 列出所有项目 """ limit = request.args.get('limit', 50, type=int) - projects = ProjectManager.list_projects(limit=limit) + actor = current_actor() + owner_user_id = actor.user_id if actor.role is Role.USER else None + projects = ProjectManager.list_projects( + limit=limit, + organization_id=actor.organization_id, + owner_user_id=owner_user_id, + ) return jsonify({ "success": True, @@ -68,11 +156,24 @@ def list_projects(): @graph_bp.route('/project/', methods=['DELETE']) +@require_auth def delete_project(project_id: str): """ 删除项目 """ - success = ProjectManager.delete_project(project_id) + project = _scoped_project(project_id) + if project is None: + return jsonify({ + "success": False, + "error": t('api.projectDeleteFailed', id=project_id) + }), 404 + + actor = current_actor() + success = ProjectManager.delete_project( + project.project_id, + organization_id=actor.organization_id, + owner_user_id=actor.user_id if actor.role is Role.USER else None, + ) if not success: return jsonify({ @@ -87,11 +188,12 @@ def delete_project(project_id: str): @graph_bp.route('/project//reset', methods=['POST']) +@require_auth def reset_project(project_id: str): """ 重置项目状态（用于重新构建图谱） """ - project = ProjectManager.get_project(project_id) + project = _scoped_project(project_id) if not project: return jsonify({ @@ -120,6 +222,8 @@ def reset_project(project_id: str): # ============== 接口1：上传文件并生成本体 ============== @graph_bp.route('/ontology/generate', methods=['POST']) +@require_auth +@idempotent def generate_ontology(): """ 接口1：上传文件，分析生成本体定义 @@ -190,9 +294,16 @@ def generate_ontology(): "error": t('api.requireFileUpload') }), 400 - # 创建项目 - project = ProjectManager.create_project(name=project_name) + # Create a tenant-owned project. + actor = current_actor() + project = ProjectManager.create_project( + name=project_name, + organization_id=actor.organization_id, + owner_user_id=actor.user_id, + ) project.simulation_requirement = simulation_requirement + # Best-effort dual-write into the durable repository. + _sync_project_to_durable(project) logger.info(f"创建项目: {project.project_id}") # 保存文件并提取文本 @@ -219,7 +330,11 @@ def generate_ontology(): all_text += f"\n\n=== {file_info['original_filename']} ===\n{text}" if not document_texts: - ProjectManager.delete_project(project.project_id) + ProjectManager.delete_project( + project.project_id, + organization_id=current_actor().organization_id, + owner_user_id=current_actor().user_id, + ) return jsonify({ "success": False, "error": t('api.noDocProcessed') @@ -252,6 +367,7 @@ def generate_ontology(): project.analysis_summary = ontology.get("analysis_summary", "") project.status = ProjectStatus.ONTOLOGY_GENERATED ProjectManager.save_project(project) + _sync_project_to_durable(project) logger.info(f"=== 本体生成完成 === 项目ID: {project.project_id}") return jsonify({ @@ -267,16 +383,15 @@ def generate_ontology(): }) except Exception as e: - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + logger.exception("Graph operation failed: %s", type(e).__name__) + raise ApiError("graph_operation_failed", 500, "api.internalError") from e # ============== 接口2：构建图谱 ============== @graph_bp.route('/build', methods=['POST']) +@require_auth +@idempotent def build_graph(): """ 接口2：根据project_id构建图谱 @@ -302,12 +417,17 @@ def build_graph(): try: logger.info("=== 开始构建图谱 ===") - # 检查配置 + # Validate the selected backend before starting an asynchronous task. + backend = Config.MEMORY_BACKEND + if backend not in {"zep", "local"}: + raise ApiError("invalid_memory_backend", 500, "api.internalError") errors = [] - if not Config.ZEP_API_KEY: + if backend == "zep" and not Config.ZEP_API_KEY: errors.append(t('api.zepApiKeyMissing')) + if backend == "local" and not Config.LLM_API_KEY: + errors.append("LLM_API_KEY not configured") if errors: - logger.error(f"配置错误: {errors}") + logger.error("Graph backend configuration is incomplete: backend=%s", backend) return jsonify({ "success": False, "error": t('api.configError', details="; ".join(errors)) @@ -325,7 +445,7 @@ def build_graph(): }), 400 # 获取项目 - project = ProjectManager.get_project(project_id) + project = _scoped_project(project_id) if not project: return jsonify({ "success": False, @@ -380,15 +500,28 @@ def build_graph(): "error": t('api.ontologyNotFound') }), 400 - # 创建异步任务 + # Create durable task metadata with tenant scope. task_manager = TaskManager() - task_id = task_manager.create_task(f"构建图谱: {graph_name}") + actor = current_actor() + organization_id = actor.organization_id + session_factory = current_app.extensions.get("crowdsight_session_factory") + if session_factory is None: + raise ApiError("memory_backend_unavailable", 503, "api.internalError") + task_id = task_manager.create_task( + f"构建图谱: {graph_name}", + metadata={ + "project_id": project_id, + "organization_id": actor.organization_id, + "owner_user_id": actor.user_id, + }, + ) logger.info(f"创建图谱构建任务: task_id={task_id}, project_id={project_id}") # 更新项目状态 project.status = ProjectStatus.GRAPH_BUILDING project.graph_build_task_id = task_id ProjectManager.save_project(project) + _sync_project_to_durable(project) # Capture locale before spawning background thread current_locale = get_locale() @@ -405,8 +538,22 @@ def build_graph(): message=t('progress.initGraphService') ) - # 创建图谱构建服务 - builder = GraphBuilderService(api_key=Config.ZEP_API_KEY) + # Choose the storage adapter inside the worker with captured scope. + if backend == "local": + builder = LocalGraphBuilderService( + session_factory, + organization_id=organization_id, + project_id=project_id, + extraction_client=LLMClient(), + language=current_locale, + ) + else: + builder = GraphBuilderService( + api_key=Config.ZEP_API_KEY, + organization_id=organization_id, + owner_user_id=actor.user_id, + session_factory=session_factory, + ) # 分块 task_manager.update_task( @@ -424,7 +571,7 @@ def build_graph(): # 创建图谱 task_manager.update_task( task_id, - message=t('progress.creatingZepGraph'), + message=t('progress.creatingGraph' if backend == "local" else 'progress.creatingZepGraph'), progress=10 ) graph_id = builder.create_graph(name=graph_name) @@ -432,6 +579,7 @@ def build_graph(): # 更新项目的graph_id project.graph_id = graph_id ProjectManager.save_project(project) + _sync_project_to_durable(project, session_factory) # 设置本体 task_manager.update_task( @@ -466,7 +614,7 @@ def build_graph(): # 等待Zep处理完成（查询每个episode的processed状态） task_manager.update_task( task_id, - message=t('progress.waitingZepProcess'), + message=t('progress.processingComplete' if backend == "local" else 'progress.waitingZepProcess'), progress=55 ) @@ -491,6 +639,7 @@ def build_graph(): # 更新项目状态 project.status = ProjectStatus.GRAPH_COMPLETED ProjectManager.save_project(project) + _sync_project_to_durable(project, session_factory) node_count = graph_data.get("node_count", 0) edge_count = graph_data.get("edge_count", 0) @@ -513,20 +662,24 @@ def build_graph(): except Exception as e: # 更新项目状态为失败 - build_logger.error(f"[{task_id}] 图谱构建失败: {str(e)}") - build_logger.debug(traceback.format_exc()) + build_logger.error( + "[%s] graph build failed: error_type=%s", + task_id, + type(e).__name__, + ) project.status = ProjectStatus.FAILED - project.error = str(e) + project.error = t('api.internalError') ProjectManager.save_project(project) + _sync_project_to_durable(project, session_factory) task_manager.update_task( task_id, status=TaskStatus.FAILED, - message=t('progress.buildFailed', error=str(e)), - error=traceback.format_exc() + message=t('api.internalError'), + error=t('api.internalError'), ) - + # 启动后台线程 thread = threading.Thread(target=build_task, daemon=True) thread.start() @@ -541,21 +694,19 @@ def build_graph(): }) except Exception as e: - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + logger.exception("Graph operation failed: %s", type(e).__name__) + raise ApiError("graph_operation_failed", 500, "api.internalError") from e # ============== 任务查询接口 ============== @graph_bp.route('/task/', methods=['GET']) +@require_auth def get_task(task_id: str): """ 查询任务状态 """ - task = TaskManager().get_task(task_id) + task = _scoped_task(task_id) if not task: return jsonify({ @@ -570,11 +721,19 @@ def get_task(task_id: str): @graph_bp.route('/tasks', methods=['GET']) +@require_auth def list_tasks(): """ 列出所有任务 """ - tasks = TaskManager().list_tasks() + actor = current_actor() + tasks = [ + task for task in TaskManager().list_tasks( + organization_id=actor.organization_id, + owner_user_id=actor.user_id if actor.role is Role.USER else None, + ) + if _scoped_task(task.task_id) is not None + ] return jsonify({ "success": True, @@ -586,18 +745,29 @@ def list_tasks(): # ============== 图谱数据接口 ============== @graph_bp.route('/data/', methods=['GET']) +@require_auth def get_graph_data(graph_id: str): """ 获取图谱数据（节点和边） """ + project = _scoped_graph(graph_id) + if project is None: + return jsonify({ + "success": False, + "error": t('api.graphNotBuilt'), + }), 404 try: - if not Config.ZEP_API_KEY: - return jsonify({ - "success": False, - "error": t('api.zepApiKeyMissing') - }), 500 - - builder = GraphBuilderService(api_key=Config.ZEP_API_KEY) + if Config.MEMORY_BACKEND == "local": + builder = _local_graph_builder(project) + elif Config.MEMORY_BACKEND == "zep": + if not Config.ZEP_API_KEY: + return jsonify({ + "success": False, + "error": t('api.zepApiKeyMissing') + }), 500 + builder = GraphBuilderService(api_key=Config.ZEP_API_KEY) + else: + raise ApiError("invalid_memory_backend", 500, "api.internalError") graph_data = builder.get_graph_data(graph_id) return jsonify({ @@ -606,26 +776,34 @@ def get_graph_data(graph_id: str): }) except Exception as e: - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + logger.exception("Graph operation failed: %s", type(e).__name__) + raise ApiError("graph_operation_failed", 500, "api.internalError") from e @graph_bp.route('/delete/', methods=['DELETE']) +@require_auth def delete_graph(graph_id: str): """ - 删除Zep图谱 + ลบกราฟ memory """ + project = _scoped_graph(graph_id) + if project is None: + return jsonify({ + "success": False, + "error": t('api.graphNotBuilt'), + }), 404 try: - if not Config.ZEP_API_KEY: - return jsonify({ - "success": False, - "error": t('api.zepApiKeyMissing') - }), 500 - - builder = GraphBuilderService(api_key=Config.ZEP_API_KEY) + if Config.MEMORY_BACKEND == "local": + builder = _local_graph_builder(project) + elif Config.MEMORY_BACKEND == "zep": + if not Config.ZEP_API_KEY: + return jsonify({ + "success": False, + "error": t('api.zepApiKeyMissing') + }), 500 + builder = GraphBuilderService(api_key=Config.ZEP_API_KEY) + else: + raise ApiError("invalid_memory_backend", 500, "api.internalError") builder.delete_graph(graph_id) return jsonify({ @@ -634,8 +812,5 @@ def delete_graph(graph_id: str): }) except Exception as e: - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + logger.exception("Graph operation failed: %s", type(e).__name__) + raise ApiError("graph_operation_failed", 500, "api.internalError") from e diff --git a/backend/app/api/report.py b/backend/app/api/report.py index fbdf460..180ea10 100644 --- a/backend/app/api/report.py +++ b/backend/app/api/report.py @@ -4,25 +4,107 @@ Report API路由 """ import os -import traceback import threading -from flask import request, jsonify, send_file +from flask import current_app, request, jsonify, send_file from . import report_bp from ..config import Config from ..services.report_agent import ReportAgent, ReportManager, ReportStatus -from ..services.simulation_manager import SimulationManager -from ..models.project import ProjectManager from ..models.task import TaskManager, TaskStatus from ..utils.logger import get_logger from ..utils.locale import t, get_locale, set_locale +from ..security.auth import authenticate_readonly_request, current_actor +from ..security.resources import enforce_request_scope, scoped_reports, require_scoped_project, require_scoped_simulation +from ..services.memory_tools import LocalMemoryTools +from ..services.product_repository import ProductRepository +from ..services.idempotency import idempotent +from ..utils.api_errors import ApiError, internal_error_payload logger = get_logger('crowdsight.api.report') +@report_bp.errorhandler(ApiError) +def handle_report_error(error: ApiError): + return jsonify(error.to_payload(t)), error.status_code + + +def _safe_internal_failure(operation: str, error: Exception): + """Return a safe response without leaking exception details.""" + if isinstance(error, ApiError): + return jsonify(error.to_payload(t)), error.status_code + logger.error("%s failed: error_type=%s", operation, type(error).__name__) + return jsonify(internal_error_payload(t)), 500 + + +def _sync_report_to_durable( + report, + *, + organization_id, + project_id, + simulation_id, + created_by_user_id, + session_factory=None, +): + """Best-effort dual-write of a legacy report into durable SQL. + + Never raises; the filesystem ReportManager remains authoritative if the + durable write is unavailable. Callers inside a worker thread pass the + captured ``session_factory`` explicitly. + """ + try: + if session_factory is None: + session_factory = current_app.extensions.get("crowdsight_session_factory") + if session_factory is None: + return + session = session_factory() + try: + ProductRepository(session).sync_report( + report, + organization_id=organization_id, + project_id=project_id, + simulation_id=simulation_id, + created_by_user_id=created_by_user_id, + commit=True, + ) + finally: + session.close() + except Exception: + logger.warning("durable report sync skipped", exc_info=True) + + +@report_bp.before_request +def _authenticate_report_request(): + authenticate_readonly_request() + enforce_request_scope() + + +def _request_memory_tools(graph_id: str): + """Build a request-scoped local memory adapter when local mode is active.""" + if Config.MEMORY_BACKEND != "local": + return None, None + + session_factory = current_app.extensions.get("crowdsight_session_factory") + if session_factory is None: + raise ApiError("memory_backend_unavailable", 503, "api.internalError") + + actor = current_actor() + session = session_factory() + try: + tools = LocalMemoryTools( + session, + organization_id=actor.organization_id, + graph_id=graph_id, + ) + except Exception: + session.close() + raise + return tools, session + + # ============== 报告生成接口 ============== @report_bp.route('/generate', methods=['POST']) +@idempotent def generate_report(): """ 生成模拟分析报告（异步任务） @@ -59,15 +141,8 @@ def generate_report(): force_regenerate = data.get('force_regenerate', False) - # 获取模拟信息 - manager = SimulationManager() - state = manager.get_simulation(simulation_id) - - if not state: - return jsonify({ - "success": False, - "error": t('api.simulationNotFound', id=simulation_id) - }), 404 + # Fetch both resources through the tenant/owner scope guards. + state = require_scoped_simulation(simulation_id) # 检查是否已有报告 if not force_regenerate: @@ -85,12 +160,7 @@ def generate_report(): }) # 获取项目信息 - project = ProjectManager.get_project(state.project_id) - if not project: - return jsonify({ - "success": False, - "error": t('api.projectNotFound', id=state.project_id) - }), 404 + project = require_scoped_project(state.project_id) graph_id = state.graph_id or project.graph_id if not graph_id: @@ -110,14 +180,26 @@ def generate_report(): import uuid report_id = f"report_{uuid.uuid4().hex[:12]}" - # 创建异步任务 + # Create an ownership-scoped in-memory task record. task_manager = TaskManager() + actor = current_actor() + organization_id = actor.organization_id + owner_user_id = actor.user_id + project_id = state.project_id + memory_backend = Config.MEMORY_BACKEND + session_factory = current_app.extensions.get("crowdsight_session_factory") if memory_backend == "local" else None + if memory_backend not in {"zep", "local"}: + raise ApiError("invalid_memory_backend", 500, "api.internalError") + if memory_backend == "local" and session_factory is None: + raise ApiError("memory_backend_unavailable", 503, "api.internalError") task_id = task_manager.create_task( task_type="report_generate", metadata={ "simulation_id": simulation_id, "graph_id": graph_id, - "report_id": report_id + "report_id": report_id, + "organization_id": actor.organization_id, + "owner_user_id": actor.user_id, } ) @@ -127,6 +209,7 @@ def generate_report(): # 定义后台任务 def run_generate(): set_locale(current_locale) + local_session = None try: task_manager.update_task( task_id, @@ -134,12 +217,23 @@ def generate_report(): progress=0, message=t('api.initReportAgent') ) - - # 创建Report Agent + + memory_tools = None + if memory_backend == "local": + if session_factory is None: + raise RuntimeError("memory_backend_unavailable") + local_session = session_factory() + memory_tools = LocalMemoryTools( + local_session, + organization_id=organization_id, + graph_id=graph_id, + ) + agent = ReportAgent( graph_id=graph_id, simulation_id=simulation_id, - simulation_requirement=simulation_requirement + simulation_requirement=simulation_requirement, + memory_tools=memory_tools, ) # 进度回调 @@ -158,6 +252,15 @@ def generate_report(): # 保存报告 ReportManager.save_report(report) + # Best-effort dual-write into the durable repository. + _sync_report_to_durable( + report, + organization_id=organization_id, + project_id=project_id, + simulation_id=simulation_id, + created_by_user_id=owner_user_id, + session_factory=session_factory, + ) if report.status == ReportStatus.COMPLETED: task_manager.complete_task( @@ -169,11 +272,14 @@ def generate_report(): } ) else: - task_manager.fail_task(task_id, report.error or t('api.reportGenerateFailed')) + task_manager.fail_task(task_id, t('api.reportGenerateFailed')) - except Exception as e: - logger.error(f"报告生成失败: {str(e)}") - task_manager.fail_task(task_id, str(e)) + except Exception as exc: + logger.error("Report generation failed: error=%s", type(exc).__name__) + task_manager.fail_task(task_id, t('api.reportGenerateFailed')) + finally: + if local_session is not None: + local_session.close() # 启动后台线程 thread = threading.Thread(target=run_generate, daemon=True) @@ -192,12 +298,7 @@ def generate_report(): }) except Exception as e: - logger.error(f"启动报告生成任务失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("start report generation", e) @report_bp.route('/generate/status', methods=['POST']) @@ -265,11 +366,7 @@ def get_generate_status(): }) except Exception as e: - logger.error(f"查询任务状态失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e) - }), 500 + return _safe_internal_failure("get report task status", e) # ============== 报告获取接口 ============== @@ -308,12 +405,7 @@ def get_report(report_id: str): }) except Exception as e: - logger.error(f"获取报告失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("get report", e) @report_bp.route('/by-simulation/', methods=['GET']) @@ -347,12 +439,7 @@ def get_report_by_simulation(simulation_id: str): }) except Exception as e: - logger.error(f"获取报告失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("get report", e) @report_bp.route('/list', methods=['GET']) @@ -375,9 +462,9 @@ def list_reports(): simulation_id = request.args.get('simulation_id') limit = request.args.get('limit', 50, type=int) - reports = ReportManager.list_reports( + reports = scoped_reports( simulation_id=simulation_id, - limit=limit + limit=limit, ) return jsonify({ @@ -387,12 +474,7 @@ def list_reports(): }) except Exception as e: - logger.error(f"列出报告失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("list reports", e) @report_bp.route('//download', methods=['GET']) @@ -433,12 +515,7 @@ def download_report(report_id: str): ) except Exception as e: - logger.error(f"下载报告失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("download report", e) @report_bp.route('/', methods=['DELETE']) @@ -459,17 +536,13 @@ def delete_report(report_id: str): }) except Exception as e: - logger.error(f"删除报告失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("delete report", e) # ============== Report Agent对话接口 ============== @report_bp.route('/chat', methods=['POST']) +@idempotent def chat_with_report_agent(): """ 与Report Agent对话 @@ -515,22 +588,9 @@ def chat_with_report_agent(): "error": t('api.requireMessage') }), 400 - # 获取模拟和项目信息 - manager = SimulationManager() - state = manager.get_simulation(simulation_id) - - if not state: - return jsonify({ - "success": False, - "error": t('api.simulationNotFound', id=simulation_id) - }), 404 - - project = ProjectManager.get_project(state.project_id) - if not project: - return jsonify({ - "success": False, - "error": t('api.projectNotFound', id=state.project_id) - }), 404 + # Resolve both resources through tenant/owner scope guards. + state = require_scoped_simulation(simulation_id) + project = require_scoped_project(state.project_id) graph_id = state.graph_id or project.graph_id if not graph_id: @@ -541,27 +601,25 @@ def chat_with_report_agent(): simulation_requirement = project.simulation_requirement or "" - # 创建Agent并进行对话 - agent = ReportAgent( - graph_id=graph_id, - simulation_id=simulation_id, - simulation_requirement=simulation_requirement - ) - - result = agent.chat(message=message, chat_history=chat_history) - - return jsonify({ - "success": True, - "data": result - }) + memory_tools, local_session = _request_memory_tools(graph_id) + try: + agent = ReportAgent( + graph_id=graph_id, + simulation_id=simulation_id, + simulation_requirement=simulation_requirement, + memory_tools=memory_tools, + ) + result = agent.chat(message=message, chat_history=chat_history) + return jsonify({ + "success": True, + "data": result + }) + finally: + if local_session is not None: + local_session.close() except Exception as e: - logger.error(f"对话失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("chat with report agent", e) # ============== 报告进度与分章节接口 ============== @@ -599,12 +657,7 @@ def get_report_progress(report_id: str): }) except Exception as e: - logger.error(f"获取报告进度失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("get report progress", e) @report_bp.route('//sections', methods=['GET']) @@ -650,12 +703,7 @@ def get_report_sections(report_id: str): }) except Exception as e: - logger.error(f"获取章节列表失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("get report sections", e) @report_bp.route('//section/', methods=['GET']) @@ -694,12 +742,7 @@ def get_single_section(report_id: str, section_index: int): }) except Exception as e: - logger.error(f"获取章节内容失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("get report section", e) # ============== 报告状态检查接口 ============== @@ -745,12 +788,7 @@ def check_report_status(simulation_id: str): }) except Exception as e: - logger.error(f"检查报告状态失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("check report status", e) # ============== Agent 日志接口 ============== @@ -806,12 +844,7 @@ def get_agent_log(report_id: str): }) except Exception as e: - logger.error(f"获取Agent日志失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("get agent log", e) @report_bp.route('//agent-log/stream', methods=['GET']) @@ -840,12 +873,7 @@ def stream_agent_log(report_id: str): }) except Exception as e: - logger.error(f"获取Agent日志失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("get agent log", e) # ============== 控制台日志接口 ============== @@ -888,12 +916,7 @@ def get_console_log(report_id: str): }) except Exception as e: - logger.error(f"获取控制台日志失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("get console log", e) @report_bp.route('//console-log/stream', methods=['GET']) @@ -922,12 +945,7 @@ def stream_console_log(report_id: str): }) except Exception as e: - logger.error(f"获取控制台日志失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("get console log", e) # ============== 工具调用接口（供调试使用）============== @@ -957,14 +975,19 @@ def search_graph_tool(): "error": t('api.requireGraphIdAndQuery') }), 400 - from ..services.zep_tools import ZepToolsService - - tools = ZepToolsService() - result = tools.search_graph( - graph_id=graph_id, - query=query, - limit=limit - ) + tools, local_session = _request_memory_tools(graph_id) + if tools is None: + from ..services.zep_tools import ZepToolsService + tools = ZepToolsService() + try: + result = tools.search_graph( + graph_id=graph_id, + query=query, + limit=limit + ) + finally: + if local_session is not None: + local_session.close() return jsonify({ "success": True, @@ -972,12 +995,7 @@ def search_graph_tool(): }) except Exception as e: - logger.error(f"图谱搜索失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("search graph tool", e) @report_bp.route('/tools/statistics', methods=['POST']) @@ -1001,10 +1019,15 @@ def get_graph_statistics_tool(): "error": t('api.requireGraphId') }), 400 - from ..services.zep_tools import ZepToolsService - - tools = ZepToolsService() - result = tools.get_graph_statistics(graph_id) + tools, local_session = _request_memory_tools(graph_id) + if tools is None: + from ..services.zep_tools import ZepToolsService + tools = ZepToolsService() + try: + result = tools.get_graph_statistics(graph_id) + finally: + if local_session is not None: + local_session.close() return jsonify({ "success": True, @@ -1012,9 +1035,4 @@ def get_graph_statistics_tool(): }) except Exception as e: - logger.error(f"获取图谱统计失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("get graph statistics tool", e) diff --git a/backend/app/api/simulation.py b/backend/app/api/simulation.py index 3a91391..8d7ea0b 100644 --- a/backend/app/api/simulation.py +++ b/backend/app/api/simulation.py @@ -5,22 +5,100 @@ Step2: Zep实体读取与过滤、OASIS模拟准备与运行（全程自动化 import os import json -import traceback -from flask import request, jsonify, send_file +from flask import current_app, request, jsonify, send_file from . import simulation_bp from ..config import Config -from ..services.zep_entity_reader import ZepEntityReader from ..services.oasis_profile_generator import OasisProfileGenerator from ..services.simulation_manager import SimulationManager, SimulationStatus from ..services.simulation_runner import SimulationRunner, RunnerStatus from ..utils.logger import get_logger from ..utils.locale import t, get_locale, set_locale from ..models.project import ProjectManager +from ..security.auth import authenticate_readonly_request, current_actor +from ..security.resources import ( + enforce_request_scope, + require_scoped_simulation, + scoped_project, + scoped_simulations, +) +from ..services.memory_entity_reader import make_local_entity_reader_factory +from ..services.memory_tools import LocalMemoryTools +from ..services.product_repository import ProductRepository +from ..services.idempotency import idempotent +from ..utils.api_errors import ApiError, internal_error_payload logger = get_logger('crowdsight.api.simulation') +def _safe_internal_failure(operation: str, error: Exception): + if isinstance(error, ApiError): + return jsonify(error.to_payload(t)), error.status_code + logger.error("%s failed: error_type=%s", operation, type(error).__name__) + return jsonify(internal_error_payload(t)), 500 + + +def _safe_client_failure(status_code: int, code: str = "invalid_request"): + return jsonify( + ApiError(code, status_code, "api.requestError").to_payload(t) + ), status_code + + +def _sync_simulation_to_durable(state, *, organization_id, project_id, created_by_user_id): + """Best-effort dual-write of a legacy simulation state into durable SQL. + + Never raises; the filesystem manager remains authoritative if the durable + write is unavailable. The SimulationState carries no tenant id, so the + caller supplies organization/project/creator which are merged in. + """ + try: + session_factory = current_app.extensions.get("crowdsight_session_factory") + if session_factory is None: + return + payload = state.to_dict() if hasattr(state, "to_dict") else dict(state) + payload["organization_id"] = organization_id + payload["project_id"] = project_id + payload["created_by_user_id"] = created_by_user_id + session = session_factory() + try: + ProductRepository(session).sync_simulation(payload, commit=True) + finally: + session.close() + except Exception: + logger.warning("durable simulation sync skipped", exc_info=True) + + +def _simulation_manager_for_request() -> SimulationManager: + backend = Config.MEMORY_BACKEND + if backend == "zep": + return SimulationManager() + if backend != "local": + raise ApiError("invalid_memory_backend", 500, "api.internalError") + session_factory = current_app.extensions.get("crowdsight_session_factory") + if session_factory is None: + raise ApiError("memory_backend_unavailable", 503, "api.internalError") + reader_factory = make_local_entity_reader_factory( + session_factory, + organization_id=current_actor().organization_id, + ) + return SimulationManager(entity_reader_factory=reader_factory) + + +def _entity_reader_for_request(graph_id: str): + return _simulation_manager_for_request().create_entity_reader(graph_id) + + +@simulation_bp.errorhandler(ApiError) +def handle_simulation_error(error: ApiError): + return jsonify(error.to_payload(t)), error.status_code + + +@simulation_bp.before_request +def _authenticate_simulation_request(): + authenticate_readonly_request() + enforce_request_scope() + + # Interview prompt 优化前缀 # 添加此前缀可以避免Agent调用工具，直接用文本回复 INTERVIEW_PROMPT_PREFIX = "结合你的人设、所有的过往记忆与行动，不调用任何工具直接用文本回复我：" @@ -58,7 +136,7 @@ def get_graph_entities(graph_id: str): enrich: 是否获取相关边信息（默认true） """ try: - if not Config.ZEP_API_KEY: + if Config.MEMORY_BACKEND == "zep" and not Config.ZEP_API_KEY: return jsonify({ "success": False, "error": t('api.zepApiKeyMissing') @@ -70,12 +148,17 @@ def get_graph_entities(graph_id: str): logger.info(f"获取图谱实体: graph_id={graph_id}, entity_types={entity_types}, enrich={enrich}") - reader = ZepEntityReader() - result = reader.filter_defined_entities( - graph_id=graph_id, - defined_entity_types=entity_types, - enrich_with_edges=enrich - ) + reader = _entity_reader_for_request(graph_id) + try: + result = reader.filter_defined_entities( + graph_id=graph_id, + defined_entity_types=entity_types, + enrich_with_edges=enrich, + ) + finally: + close_reader = getattr(reader, "close", None) + if callable(close_reader): + close_reader() return jsonify({ "success": True, @@ -83,26 +166,26 @@ def get_graph_entities(graph_id: str): }) except Exception as e: - logger.error(f"获取图谱实体失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('/entities//', methods=['GET']) def get_entity_detail(graph_id: str, entity_uuid: str): """获取单个实体的详细信息""" try: - if not Config.ZEP_API_KEY: + if Config.MEMORY_BACKEND == "zep" and not Config.ZEP_API_KEY: return jsonify({ "success": False, "error": t('api.zepApiKeyMissing') }), 500 - reader = ZepEntityReader() - entity = reader.get_entity_with_context(graph_id, entity_uuid) + reader = _entity_reader_for_request(graph_id) + try: + entity = reader.get_entity_with_context(graph_id, entity_uuid) + finally: + close_reader = getattr(reader, "close", None) + if callable(close_reader): + close_reader() if not entity: return jsonify({ @@ -116,19 +199,14 @@ def get_entity_detail(graph_id: str, entity_uuid: str): }) except Exception as e: - logger.error(f"获取实体详情失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('/entities//by-type/', methods=['GET']) def get_entities_by_type(graph_id: str, entity_type: str): """获取指定类型的所有实体""" try: - if not Config.ZEP_API_KEY: + if Config.MEMORY_BACKEND == "zep" and not Config.ZEP_API_KEY: return jsonify({ "success": False, "error": t('api.zepApiKeyMissing') @@ -136,12 +214,17 @@ def get_entities_by_type(graph_id: str, entity_type: str): enrich = request.args.get('enrich', 'true').lower() == 'true' - reader = ZepEntityReader() - entities = reader.get_entities_by_type( - graph_id=graph_id, - entity_type=entity_type, - enrich_with_edges=enrich - ) + reader = _entity_reader_for_request(graph_id) + try: + entities = reader.get_entities_by_type( + graph_id=graph_id, + entity_type=entity_type, + enrich_with_edges=enrich, + ) + finally: + close_reader = getattr(reader, "close", None) + if callable(close_reader): + close_reader() return jsonify({ "success": True, @@ -153,17 +236,13 @@ def get_entities_by_type(graph_id: str, entity_type: str): }) except Exception as e: - logger.error(f"获取实体失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) # ============== 模拟管理接口 ============== @simulation_bp.route('/create', methods=['POST']) +@idempotent def create_simulation(): """ 创建新的模拟 @@ -202,27 +281,51 @@ def create_simulation(): "error": t('api.requireProjectId') }), 400 - project = ProjectManager.get_project(project_id) + project = scoped_project(project_id) if not project: return jsonify({ "success": False, "error": t('api.projectNotFound', id=project_id) }), 404 - + graph_id = data.get('graph_id') or project.graph_id if not graph_id: return jsonify({ "success": False, "error": t('api.graphNotBuilt') }), 400 + + actor = current_actor() + graph_project = ProjectManager.find_project_by_graph_id( + graph_id, + organization_id=actor.organization_id, + owner_user_id=actor.user_id, + ) + if graph_project is None or graph_project.project_id != project.project_id: + return jsonify({ + "success": False, + "error": t('api.graphNotBuilt') + }), 404 - manager = SimulationManager() + manager = SimulationManager( + session_factory=current_app.extensions.get("crowdsight_session_factory") + ) state = manager.create_simulation( project_id=project_id, graph_id=graph_id, enable_twitter=data.get('enable_twitter', True), enable_reddit=data.get('enable_reddit', True), ) + # Attach tenant scope so every subsequently saved state mirrors durable. + state.organization_id = actor.organization_id + state.owner_user_id = actor.user_id + # Best-effort dual-write into the durable repository. + _sync_simulation_to_durable( + state, + organization_id=actor.organization_id, + project_id=project_id, + created_by_user_id=actor.user_id, + ) return jsonify({ "success": True, @@ -230,12 +333,7 @@ def create_simulation(): }) except Exception as e: - logger.error(f"创建模拟失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) def _check_simulation_prepared(simulation_id: str) -> tuple: @@ -332,7 +430,7 @@ def _check_simulation_prepared(simulation_id: str) -> tuple: logger.info(f"自动更新模拟状态: {simulation_id} preparing -> ready") status = "ready" except Exception as e: - logger.warning(f"自动更新状态失败: {e}") + logger.warning("自动更新状态失败: error_type=%s", type(e).__name__) logger.info(f"模拟 {simulation_id} 检测结果: 已准备完成 (status={status}, config_generated={config_generated})") return True, { @@ -354,10 +452,12 @@ def _check_simulation_prepared(simulation_id: str) -> tuple: } except Exception as e: - return False, {"reason": f"读取状态文件失败: {str(e)}"} + logger.warning("读取状态文件失败: error_type=%s", type(e).__name__) + return False, {"reason": "preparation_status_unavailable"} @simulation_bp.route('/prepare', methods=['POST']) +@idempotent def prepare_simulation(): """ 准备模拟环境（异步任务，LLM智能生成所有参数） @@ -413,7 +513,7 @@ def prepare_simulation(): "error": t('api.requireSimulationId') }), 400 - manager = SimulationManager() + manager = _simulation_manager_for_request() state = manager.get_simulation(simulation_id) if not state: @@ -422,9 +522,8 @@ def prepare_simulation(): "error": t('api.simulationNotFound', id=simulation_id) }), 404 - # 检查是否强制重新生成 + # Check whether a forced regeneration was requested. force_regenerate = data.get('force_regenerate', False) - logger.info(f"开始处理 /prepare 请求: simulation_id={simulation_id}, force_regenerate={force_regenerate}") # 检查是否已经准备完成（避免重复生成） if not force_regenerate: @@ -473,13 +572,18 @@ def prepare_simulation(): # 这样前端在调用prepare后立即就能获取到预期Agent总数 try: logger.info(f"同步获取实体数量: graph_id={state.graph_id}") - reader = ZepEntityReader() - # 快速读取实体（不需要边信息，只统计数量） - filtered_preview = reader.filter_defined_entities( - graph_id=state.graph_id, - defined_entity_types=entity_types_list, - enrich_with_edges=False # 不获取边信息，加快速度 - ) + reader = manager.create_entity_reader(state.graph_id) + try: + # Quick read without edge enrichment for immediate progress counts. + filtered_preview = reader.filter_defined_entities( + graph_id=state.graph_id, + defined_entity_types=entity_types_list, + enrich_with_edges=False, + ) + finally: + close_reader = getattr(reader, "close", None) + if callable(close_reader): + close_reader() # 保存实体数量到状态（供前端立即获取） state.entities_count = filtered_preview.filtered_count state.entity_types = list(filtered_preview.entity_types) @@ -490,11 +594,15 @@ def prepare_simulation(): # 创建异步任务 task_manager = TaskManager() + actor = current_actor() task_id = task_manager.create_task( task_type="simulation_prepare", metadata={ "simulation_id": simulation_id, - "project_id": state.project_id + "project_id": state.project_id, + "graph_id": state.graph_id, + "organization_id": actor.organization_id, + "owner_user_id": actor.user_id, } ) @@ -598,14 +706,14 @@ def prepare_simulation(): ) except Exception as e: - logger.error(f"准备模拟失败: {str(e)}") - task_manager.fail_task(task_id, str(e)) + logger.error("准备模拟失败: error_type=%s", type(e).__name__) + task_manager.fail_task(task_id, t('api.internalError')) # 更新模拟状态为失败 state = manager.get_simulation(simulation_id) if state: state.status = SimulationStatus.FAILED - state.error = str(e) + state.error = t('api.internalError') manager._save_simulation_state(state) # 启动后台线程 @@ -625,19 +733,11 @@ def prepare_simulation(): } }) - except ValueError as e: - return jsonify({ - "success": False, - "error": str(e) - }), 404 + except ValueError: + return _safe_client_failure(404) except Exception as e: - logger.error(f"启动准备任务失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('/prepare/status', methods=['POST']) @@ -746,11 +846,7 @@ def get_prepare_status(): }) except Exception as e: - logger.error(f"查询任务状态失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e) - }), 500 + return _safe_internal_failure("prepare task status", e) @simulation_bp.route('/', methods=['GET']) @@ -758,13 +854,7 @@ def get_simulation(simulation_id: str): """获取模拟状态""" try: manager = SimulationManager() - state = manager.get_simulation(simulation_id) - - if not state: - return jsonify({ - "success": False, - "error": t('api.simulationNotFound', id=simulation_id) - }), 404 + state = require_scoped_simulation(simulation_id) result = state.to_dict() @@ -778,12 +868,7 @@ def get_simulation(simulation_id: str): }) except Exception as e: - logger.error(f"获取模拟状态失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('/list', methods=['GET']) @@ -797,8 +882,7 @@ def list_simulations(): try: project_id = request.args.get('project_id') - manager = SimulationManager() - simulations = manager.list_simulations(project_id=project_id) + simulations = scoped_simulations(project_id=project_id) return jsonify({ "success": True, @@ -807,12 +891,7 @@ def list_simulations(): }) except Exception as e: - logger.error(f"列出模拟失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) def _get_report_id_for_simulation(simulation_id: str) -> str: @@ -913,7 +992,7 @@ def get_simulation_history(): limit = request.args.get('limit', 20, type=int) manager = SimulationManager() - simulations = manager.list_simulations()[:limit] + simulations = scoped_simulations()[:limit] # 增强模拟数据，只从 Simulation 文件读取 enriched_simulations = [] @@ -980,12 +1059,7 @@ def get_simulation_history(): }) except Exception as e: - logger.error(f"获取历史模拟失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('//profiles', methods=['GET']) @@ -1011,19 +1085,11 @@ def get_simulation_profiles(simulation_id: str): } }) - except ValueError as e: - return jsonify({ - "success": False, - "error": str(e) - }), 404 + except ValueError: + return _safe_client_failure(404) except Exception as e: - logger.error(f"获取Profile失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('//profiles/realtime', methods=['GET']) @@ -1128,12 +1194,7 @@ def get_simulation_profiles_realtime(simulation_id: str): }) except Exception as e: - logger.error(f"实时获取Profile失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('//config/realtime', methods=['GET']) @@ -1248,12 +1309,7 @@ def get_simulation_config_realtime(simulation_id: str): }) except Exception as e: - logger.error(f"实时获取Config失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('//config', methods=['GET']) @@ -1284,12 +1340,7 @@ def get_simulation_config(simulation_id: str): }) except Exception as e: - logger.error(f"获取配置失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('//config/download', methods=['GET']) @@ -1313,12 +1364,7 @@ def download_simulation_config(simulation_id: str): ) except Exception as e: - logger.error(f"下载配置失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('/script//download', methods=['GET']) @@ -1365,17 +1411,13 @@ def download_simulation_script(script_name: str): ) except Exception as e: - logger.error(f"下载脚本失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) # ============== Profile生成接口（独立使用） ============== @simulation_bp.route('/generate-profiles', methods=['POST']) +@idempotent def generate_profiles(): """ 直接从图谱生成OASIS Agent Profile（不创建模拟） @@ -1402,24 +1444,40 @@ def generate_profiles(): use_llm = data.get('use_llm', True) platform = data.get('platform', 'reddit') - reader = ZepEntityReader() - filtered = reader.filter_defined_entities( - graph_id=graph_id, - defined_entity_types=entity_types, - enrich_with_edges=True - ) - - if filtered.filtered_count == 0: - return jsonify({ - "success": False, - "error": t('api.noMatchingEntities') - }), 400 - - generator = OasisProfileGenerator() - profiles = generator.generate_profiles_from_entities( - entities=filtered.entities, - use_llm=use_llm - ) + reader = _entity_reader_for_request(graph_id) + try: + filtered = reader.filter_defined_entities( + graph_id=graph_id, + defined_entity_types=entity_types, + enrich_with_edges=True + ) + + if filtered.filtered_count == 0: + return jsonify({ + "success": False, + "error": t('api.noMatchingEntities') + }), 400 + + local_memory_tools = None + if Config.MEMORY_BACKEND == "local": + repository = getattr(reader, "repository", None) + if repository is None: + raise ApiError("local_memory_reader_required", 500, "api.internalError") + local_memory_tools = LocalMemoryTools(repository) + + generator = OasisProfileGenerator( + graph_id=graph_id, + use_zep_context=Config.MEMORY_BACKEND != "local", + local_memory_tools=local_memory_tools, + ) + profiles = generator.generate_profiles_from_entities( + entities=filtered.entities, + use_llm=use_llm + ) + finally: + close_reader = getattr(reader, "close", None) + if callable(close_reader): + close_reader() if platform == "reddit": profiles_data = [p.to_reddit_format() for p in profiles] @@ -1439,12 +1497,7 @@ def generate_profiles(): }) except Exception as e: - logger.error(f"生成Profile失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) # ============== 模拟运行控制接口 ============== @@ -1485,6 +1538,7 @@ def _filter_simulation_agents(simulation_id: str, selected_agent_ids: list): logger.info(f"Twitter profiles: {len(rows)} -> {len(filtered)}") @simulation_bp.route('/start', methods=['POST']) +@idempotent def start_simulation(): """ 开始运行模拟 @@ -1592,7 +1646,7 @@ def start_simulation(): try: SimulationRunner.stop_simulation(simulation_id) except Exception as e: - logger.warning(f"停止模拟时出现警告: {str(e)}") + logger.warning("停止模拟时出现警告: error_type=%s", type(e).__name__) else: return jsonify({ "success": False, @@ -1648,7 +1702,9 @@ def start_simulation(): platform=platform, max_rounds=max_rounds, enable_graph_memory_update=enable_graph_memory_update, - graph_id=graph_id + graph_id=graph_id, + organization_id=current_actor().organization_id if Config.MEMORY_BACKEND == "local" else None, + session_factory=current_app.extensions.get("crowdsight_session_factory") if Config.MEMORY_BACKEND == "local" else None, ) # 更新模拟状态 @@ -1668,19 +1724,11 @@ def start_simulation(): "data": response_data }) - except ValueError as e: - return jsonify({ - "success": False, - "error": str(e) - }), 400 + except ValueError: + return _safe_client_failure(400) except Exception as e: - logger.error(f"启动模拟失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('/stop', methods=['POST']) @@ -1727,19 +1775,11 @@ def stop_simulation(): "data": run_state.to_dict() }) - except ValueError as e: - return jsonify({ - "success": False, - "error": str(e) - }), 400 + except ValueError: + return _safe_client_failure(400) except Exception as e: - logger.error(f"停止模拟失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) # ============== 实时状态监控接口 ============== @@ -1794,12 +1834,7 @@ def get_run_status(simulation_id: str): }) except Exception as e: - logger.error(f"获取运行状态失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('//run-status/detail', methods=['GET']) @@ -1895,12 +1930,7 @@ def get_run_status_detail(simulation_id: str): }) except Exception as e: - logger.error(f"获取详细状态失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('//actions', methods=['GET']) @@ -1949,12 +1979,7 @@ def get_simulation_actions(simulation_id: str): }) except Exception as e: - logger.error(f"获取动作历史失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('//timeline', methods=['GET']) @@ -1989,12 +2014,7 @@ def get_simulation_timeline(simulation_id: str): }) except Exception as e: - logger.error(f"获取时间线失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('//agent-stats', methods=['GET']) @@ -2016,12 +2036,7 @@ def get_agent_stats(simulation_id: str): }) except Exception as e: - logger.error(f"获取Agent统计失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) # ============== 数据库查询接口 ============== @@ -2096,12 +2111,7 @@ def get_simulation_posts(simulation_id: str): }) except Exception as e: - logger.error(f"获取帖子失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('//comments', methods=['GET']) @@ -2171,12 +2181,7 @@ def get_simulation_comments(simulation_id: str): }) except Exception as e: - logger.error(f"获取评论失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) # ============== Interview 采访接口 ============== @@ -2289,25 +2294,14 @@ def interview_agent(): "data": result }) - except ValueError as e: - return jsonify({ - "success": False, - "error": str(e) - }), 400 + except ValueError: + return _safe_client_failure(400) - except TimeoutError as e: - return jsonify({ - "success": False, - "error": t('api.interviewTimeout', error=str(e)) - }), 504 + except TimeoutError: + return _safe_client_failure(504, "interview_timeout") except Exception as e: - logger.error(f"Interview失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('/interview/batch', methods=['POST']) @@ -2427,25 +2421,14 @@ def interview_agents_batch(): "data": result }) - except ValueError as e: - return jsonify({ - "success": False, - "error": str(e) - }), 400 + except ValueError: + return _safe_client_failure(400) - except TimeoutError as e: - return jsonify({ - "success": False, - "error": t('api.batchInterviewTimeout', error=str(e)) - }), 504 + except TimeoutError: + return _safe_client_failure(504, "batch_interview_timeout") except Exception as e: - logger.error(f"批量Interview失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('/interview/all', methods=['POST']) @@ -2530,25 +2513,14 @@ def interview_all_agents(): "data": result }) - except ValueError as e: - return jsonify({ - "success": False, - "error": str(e) - }), 400 + except ValueError: + return _safe_client_failure(400) - except TimeoutError as e: - return jsonify({ - "success": False, - "error": t('api.globalInterviewTimeout', error=str(e)) - }), 504 + except TimeoutError: + return _safe_client_failure(504, "global_interview_timeout") except Exception as e: - logger.error(f"全局Interview失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('/interview/history', methods=['POST']) @@ -2615,12 +2587,7 @@ def get_interview_history(): }) except Exception as e: - logger.error(f"获取Interview历史失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('/env-status', methods=['POST']) @@ -2680,12 +2647,7 @@ def get_env_status(): }) except Exception as e: - logger.error(f"获取环境状态失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) @simulation_bp.route('/close-env', methods=['POST']) @@ -2743,16 +2705,8 @@ def close_simulation_env(): "data": result }) - except ValueError as e: - return jsonify({ - "success": False, - "error": str(e) - }), 400 + except ValueError: + return _safe_client_failure(400) except Exception as e: - logger.error(f"关闭环境失败: {str(e)}") - return jsonify({ - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + return _safe_internal_failure("simulation operation", e) diff --git a/backend/app/api/template.py b/backend/app/api/template.py index b20be28..a100196 100644 --- a/backend/app/api/template.py +++ b/backend/app/api/template.py @@ -9,6 +9,9 @@ from flask import Blueprint, request, jsonify from ..utils.llm_client import LLMClient from ..utils.locale import t, get_locale, get_language_instruction from ..utils.logger import get_logger +from ..utils.api_errors import internal_error_payload +from ..security.auth import require_auth +from ..services.idempotency import idempotent logger = get_logger('crowdsight.template') @@ -22,6 +25,7 @@ with open(_templates_path, 'r', encoding='utf-8') as f: @template_bp.route('/list', methods=['GET']) +@require_auth def list_templates(): """Return all available templates""" locale = get_locale() @@ -42,6 +46,8 @@ def list_templates(): @template_bp.route('/auto-select', methods=['POST']) +@require_auth +@idempotent def auto_select_template(): """ Analyze seed data and recommend the best template + pre-fill prompt. @@ -129,11 +135,12 @@ Return JSON: }) except Exception as e: - logger.error(f"Template auto-select failed: {e}") - return jsonify({'success': False, 'error': str(e)}), 500 + logger.error("Template auto-select failed: error_type=%s", type(e).__name__) + return jsonify(internal_error_payload(t)), 500 @template_bp.route('//filter-rules', methods=['GET']) +@require_auth def get_filter_rules(template_id): """Return entity filter rules for a template (used by Step 1)""" for tmpl in _templates: diff --git a/backend/app/config.py b/backend/app/config.py index f3608c8..a4e9e05 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -61,9 +61,16 @@ _llm_base_url, _llm_model_name, _llm_provider = _resolve_llm_config() class Config: """Flask配置类""" - # Flask配置 - SECRET_KEY = os.environ.get('SECRET_KEY', 'crowdsight-secret-key') - DEBUG = os.environ.get('FLASK_DEBUG', 'True').lower() == 'true' + # Flask configuration: production-safe defaults; secrets must be supplied. + SECRET_KEY = os.environ.get('SECRET_KEY') + DEBUG = os.environ.get('FLASK_DEBUG', 'False').lower() == 'true' + # Session cookies must be Secure in production; local HTTP can opt out explicitly. + SESSION_COOKIE_SECURE = os.environ.get('SESSION_COOKIE_SECURE', 'False').lower() == 'true' + CORS_ALLOWED_ORIGINS = [ + origin.strip() + for origin in os.environ.get('CORS_ALLOWED_ORIGINS', 'http://localhost:3000').split(',') + if origin.strip() + ] # JSON配置 - 禁用ASCII转义，让中文直接显示（而不是 \uXXXX 格式） JSON_AS_ASCII = False @@ -74,7 +81,8 @@ class Config: LLM_BASE_URL = _llm_base_url LLM_MODEL_NAME = _llm_model_name - # Zep配置 + # Memory backend migration switch: keep Zep as the explicit default until data parity/cutover is complete. + MEMORY_BACKEND = os.environ.get('MEMORY_BACKEND', 'zep').strip().lower() ZEP_API_KEY = os.environ.get('ZEP_API_KEY') # 文件上传配置 @@ -109,10 +117,14 @@ class Config: def validate(cls) -> list[str]: """验证必要配置""" errors: list[str] = [] + if not cls.SECRET_KEY: + errors.append("SECRET_KEY not configured") if not cls.LLM_API_KEY: - errors.append("LLM_API_KEY 未配置") - if not cls.ZEP_API_KEY: - errors.append("ZEP_API_KEY 未配置") + errors.append("LLM_API_KEY not configured") + if cls.MEMORY_BACKEND not in {"zep", "local"}: + errors.append("MEMORY_BACKEND must be zep or local") + elif cls.MEMORY_BACKEND == "zep" and not cls.ZEP_API_KEY: + errors.append("ZEP_API_KEY not configured") return errors @classmethod diff --git a/backend/app/db.py b/backend/app/db.py new file mode 100644 index 0000000..232571d --- /dev/null +++ b/backend/app/db.py @@ -0,0 +1,50 @@ +"""Database engine and declarative base helpers.""" + +from __future__ import annotations + +import os +from pathlib import Path + +from sqlalchemy import create_engine, event +from sqlalchemy.engine import Engine +from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker + + +class Base(DeclarativeBase): + pass + + +def create_database_engine(database_url: str | None = None, **kwargs) -> Engine: + """Create a configured SQLAlchemy engine without opening a global session.""" + url = database_url or os.environ.get("DATABASE_URL") + if not url: + data_dir = Path(os.environ.get("CROWDSIGHT_DATA_DIR", "backend/uploads")) + data_dir.mkdir(parents=True, exist_ok=True) + url = f"sqlite+pysqlite:///{(data_dir / 'crowdsight.db').resolve()}" + + connect_args = dict(kwargs.pop("connect_args", {})) + if url.startswith("sqlite"): + connect_args.setdefault("check_same_thread", False) + + engine = create_engine( + url, + future=True, + pool_pre_ping=True, + connect_args=connect_args, + **kwargs, + ) + if url.startswith("sqlite"): + @event.listens_for(engine, "connect") + def _enable_sqlite_foreign_keys(dbapi_connection, _connection_record): + cursor = dbapi_connection.cursor() + try: + cursor.execute("PRAGMA foreign_keys=ON") + finally: + cursor.close() + + return engine + + +def create_session_factory(engine: Engine) -> sessionmaker[Session]: + """Return a factory; callers own transaction boundaries and commits.""" + return sessionmaker(bind=engine, autoflush=True, expire_on_commit=False) diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 55bec61..bef1cae 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -4,6 +4,21 @@ from .task import TaskManager, TaskStatus from .project import Project, ProjectStatus, ProjectManager +from .saas import AuthSession, Membership, Organization, User +from .memory import MemoryEdge, MemoryEpisode, MemoryGraph, MemoryNode +from .operations import AuditLog, IdempotencyRecord, Job, JobStatus +from .product import DurableReport, ProductProject, ProductSimulation, ProjectStatus, ReportStatus, SimulationStatus +from .settings import PlatformSettings +from .rate_limit import RateLimitEvent +from .usage import UsageEvent +from .password_reset import PasswordResetToken -__all__ = ['TaskManager', 'TaskStatus', 'Project', 'ProjectStatus', 'ProjectManager'] +__all__ = [ + 'TaskManager', 'TaskStatus', 'Project', 'ProjectStatus', 'ProjectManager', + 'AuthSession', 'Membership', 'Organization', 'User', + 'MemoryEdge', 'MemoryEpisode', 'MemoryGraph', 'MemoryNode', + 'AuditLog', 'IdempotencyRecord', 'Job', 'JobStatus', + 'ProductProject', 'ProductSimulation', 'DurableReport', 'SimulationStatus', 'ReportStatus', + 'PlatformSettings', 'RateLimitEvent', 'UsageEvent', 'PasswordResetToken', +] diff --git a/backend/app/models/memory.py b/backend/app/models/memory.py new file mode 100644 index 0000000..c7011cd --- /dev/null +++ b/backend/app/models/memory.py @@ -0,0 +1,141 @@ +"""Durable local graph-memory schema replacing the storage side of Zep.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import uuid4 + +from sqlalchemy import DateTime, Float, ForeignKey, Index, JSON, String, Text, UniqueConstraint, func, text +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from ..db import Base + + +def _id(prefix: str) -> str: + return f"{prefix}_{uuid4().hex}" + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class MemoryGraph(Base): + __tablename__ = "memory_graphs" + + id: Mapped[str] = mapped_column(String(128), primary_key=True, default=lambda: _id("graph")) + organization_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + project_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True) + ontology: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict, server_default=text("'{}'")) + status: Mapped[str] = mapped_column(String(32), nullable=False, default="ready", server_default=text("'ready'")) + version: Mapped[int] = mapped_column(nullable=False, default=1, server_default=text("1")) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now(), onupdate=_utc_now + ) + + episodes: Mapped[list["MemoryEpisode"]] = relationship( + back_populates="graph", cascade="all, delete-orphan" + ) + nodes: Mapped[list["MemoryNode"]] = relationship( + back_populates="graph", cascade="all, delete-orphan" + ) + edges: Mapped[list["MemoryEdge"]] = relationship( + back_populates="graph", cascade="all, delete-orphan" + ) + + +class MemoryEpisode(Base): + __tablename__ = "memory_episodes" + __table_args__ = ( + UniqueConstraint("graph_id", "source_type", "source_ref", name="uq_memory_episode_source"), + Index("ix_memory_episodes_graph_status", "graph_id", "status"), + ) + + id: Mapped[str] = mapped_column(String(128), primary_key=True, default=lambda: _id("episode")) + graph_id: Mapped[str] = mapped_column( + ForeignKey("memory_graphs.id", ondelete="CASCADE"), nullable=False, index=True + ) + source_type: Mapped[str] = mapped_column(String(32), nullable=False) + source_ref: Mapped[str] = mapped_column(String(256), nullable=False) + normalized_text: Mapped[str] = mapped_column(Text, nullable=False) + summary: Mapped[str] = mapped_column(Text, nullable=False, default="", server_default=text("''")) + status: Mapped[str] = mapped_column(String(32), nullable=False, default="processed", server_default=text("'processed'")) + extractor_version: Mapped[str] = mapped_column(String(64), nullable=False, default="v1", server_default=text("'v1'")) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now() + ) + + graph: Mapped[MemoryGraph] = relationship(back_populates="episodes") + + +class MemoryNode(Base): + __tablename__ = "memory_nodes" + __table_args__ = ( + UniqueConstraint("graph_id", "normalized_name", name="uq_memory_node_graph_name"), + Index("ix_memory_nodes_graph_name", "graph_id", "normalized_name"), + ) + + id: Mapped[str] = mapped_column(String(128), primary_key=True, default=lambda: _id("node")) + graph_id: Mapped[str] = mapped_column( + ForeignKey("memory_graphs.id", ondelete="CASCADE"), nullable=False, index=True + ) + canonical_name: Mapped[str] = mapped_column(String(512), nullable=False) + normalized_name: Mapped[str] = mapped_column(String(512), nullable=False) + labels: Mapped[list] = mapped_column(JSON, nullable=False, default=list, server_default=text("'[]'")) + aliases: Mapped[list] = mapped_column(JSON, nullable=False, default=list, server_default=text("'[]'")) + attributes: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict, server_default=text("'{}'")) + summary: Mapped[str] = mapped_column(Text, nullable=False, default="", server_default=text("''")) + confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.0, server_default=text("0")) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now(), onupdate=_utc_now + ) + + graph: Mapped[MemoryGraph] = relationship(back_populates="nodes") + outgoing_edges: Mapped[list["MemoryEdge"]] = relationship( + foreign_keys="MemoryEdge.source_node_id", back_populates="source_node" + ) + incoming_edges: Mapped[list["MemoryEdge"]] = relationship( + foreign_keys="MemoryEdge.target_node_id", back_populates="target_node" + ) + + +class MemoryEdge(Base): + __tablename__ = "memory_edges" + __table_args__ = ( + Index("ix_memory_edges_graph_relation", "graph_id", "relation"), + Index("ix_memory_edges_graph_temporal", "graph_id", "valid_at", "invalid_at"), + ) + + id: Mapped[str] = mapped_column(String(128), primary_key=True, default=lambda: _id("edge")) + graph_id: Mapped[str] = mapped_column( + ForeignKey("memory_graphs.id", ondelete="CASCADE"), nullable=False, index=True + ) + source_node_id: Mapped[str] = mapped_column( + ForeignKey("memory_nodes.id", ondelete="CASCADE"), nullable=False, index=True + ) + target_node_id: Mapped[str] = mapped_column( + ForeignKey("memory_nodes.id", ondelete="CASCADE"), nullable=False, index=True + ) + relation: Mapped[str] = mapped_column(String(128), nullable=False) + fact: Mapped[str] = mapped_column(Text, nullable=False) + attributes: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict, server_default=text("'{}'")) + confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.0, server_default=text("0")) + valid_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + invalid_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + expired_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now() + ) + + graph: Mapped[MemoryGraph] = relationship(back_populates="edges") + source_node: Mapped[MemoryNode] = relationship( + foreign_keys=[source_node_id], back_populates="outgoing_edges" + ) + target_node: Mapped[MemoryNode] = relationship( + foreign_keys=[target_node_id], back_populates="incoming_edges" + ) diff --git a/backend/app/models/operations.py b/backend/app/models/operations.py new file mode 100644 index 0000000..04c38ec --- /dev/null +++ b/backend/app/models/operations.py @@ -0,0 +1,128 @@ +"""Durable operation metadata for jobs, retries, idempotency, and audit.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from enum import Enum +from uuid import uuid4 + +from sqlalchemy import JSON, CheckConstraint, DateTime, ForeignKey, Index, Integer, String, Text, UniqueConstraint, func, text +from sqlalchemy.orm import Mapped, mapped_column + +from ..db import Base + + +def _id(prefix: str) -> str: + return f"{prefix}_{uuid4().hex}" + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class JobStatus(str, Enum): + QUEUED = "queued" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + CANCELLED = "cancelled" + + +class Job(Base): + """Durable, tenant-owned unit of asynchronous work.""" + + __tablename__ = "jobs" + __table_args__ = ( + CheckConstraint( + "status IN ('queued', 'running', 'succeeded', 'failed', 'cancelled')", + name="ck_jobs_status", + ), + Index("ix_jobs_org_status_created", "organization_id", "status", "created_at"), + Index("ix_jobs_org_owner", "organization_id", "owner_user_id"), + ) + + id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: _id("job")) + organization_id: Mapped[str] = mapped_column( + ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True + ) + owner_user_id: Mapped[str | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True + ) + project_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True) + graph_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True) + operation: Mapped[str] = mapped_column(String(120), nullable=False) + status: Mapped[JobStatus] = mapped_column( + String(32), nullable=False, default=JobStatus.QUEUED.value, server_default=text("'queued'") + ) + progress: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0")) + message: Mapped[str] = mapped_column(Text, nullable=False, default="", server_default=text("''")) + result: Mapped[dict | list | None] = mapped_column(JSON, nullable=True) + progress_detail: Mapped[dict | list | None] = mapped_column(JSON, nullable=True) + job_metadata: Mapped[dict | list | None] = mapped_column("metadata", JSON, nullable=True) + error_code: Mapped[str | None] = mapped_column(String(120), nullable=True) + result_ref: Mapped[str | None] = mapped_column(String(512), nullable=True) + idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True) + attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0")) + settings_version: Mapped[str | None] = mapped_column(String(128), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utc_now, server_default=func.now(), nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utc_now, onupdate=_utc_now, server_default=func.now(), nullable=False + ) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + +class IdempotencyRecord(Base): + """Request fingerprint and replayable response for retry-safe mutations.""" + + __tablename__ = "idempotency_records" + __table_args__ = ( + UniqueConstraint("organization_id", "user_id", "key", name="uq_idempotency_org_user_key"), + Index("ix_idempotency_expiry", "expires_at"), + ) + + id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: _id("idem")) + organization_id: Mapped[str] = mapped_column( + ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True + ) + user_id: Mapped[str] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + key: Mapped[str] = mapped_column(String(128), nullable=False) + request_hash: Mapped[str] = mapped_column(String(64), nullable=False) + status: Mapped[str] = mapped_column( + String(32), nullable=False, default="reserved", server_default=text("'reserved'") + ) + response_status: Mapped[int | None] = mapped_column(Integer, nullable=True) + response_body: Mapped[dict | list | None] = mapped_column(JSON, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utc_now, server_default=func.now(), nullable=False + ) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + +class AuditLog(Base): + """Tenant-scoped redacted audit event; details must never contain secrets.""" + + __tablename__ = "audit_logs" + __table_args__ = ( + Index("ix_audit_org_created", "organization_id", "created_at"), + Index("ix_audit_org_target", "organization_id", "target_type", "target_id"), + ) + + id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: _id("audit")) + organization_id: Mapped[str] = mapped_column( + ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True + ) + actor_user_id: Mapped[str | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True + ) + action: Mapped[str] = mapped_column(String(160), nullable=False) + target_type: Mapped[str] = mapped_column(String(80), nullable=False) + target_id: Mapped[str | None] = mapped_column(String(160), nullable=True) + details: Mapped[dict | list | None] = mapped_column("metadata", JSON, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utc_now, server_default=func.now(), nullable=False + ) diff --git a/backend/app/models/password_reset.py b/backend/app/models/password_reset.py new file mode 100644 index 0000000..0de1a70 --- /dev/null +++ b/backend/app/models/password_reset.py @@ -0,0 +1,37 @@ +"""Durable, single-use password reset tokens.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import uuid4 + +from sqlalchemy import Boolean, DateTime, ForeignKey, String, func, text +from sqlalchemy.orm import Mapped, mapped_column + +from ..db import Base + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class PasswordResetToken(Base): + """One hash of a one-time reset token; plaintext is never stored.""" + + __tablename__ = "password_reset_tokens" + + id: Mapped[str] = mapped_column( + String(64), primary_key=True, default=lambda: f"prt_{uuid4().hex}" + ) + user_id: Mapped[str] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + token_hash: Mapped[str] = mapped_column(String(128), nullable=False) + auth_version: Mapped[int] = mapped_column(nullable=False, default=0, server_default=text("0")) + used: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False, server_default=text("0") + ) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now() + ) diff --git a/backend/app/models/product.py b/backend/app/models/product.py new file mode 100644 index 0000000..9793021 --- /dev/null +++ b/backend/app/models/product.py @@ -0,0 +1,186 @@ +"""Durable product-resource schema: projects, simulations, and reports. + +These replace the legacy filesystem-backed ProjectManager / SimulationManager / +ReportManager payloads with tenant- and owner-scoped SQL rows, so product state +survives restarts and multiple workers without cross-tenant leakage. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from enum import Enum +from uuid import uuid4 + +from sqlalchemy import ( + DateTime, + ForeignKey, + Index, + Integer, + JSON, + String, + Text, + UniqueConstraint, + func, + text, +) +from sqlalchemy.orm import Mapped, mapped_column + +from ..db import Base + + +def _id(prefix: str) -> str: + return f"{prefix}_{uuid4().hex}" + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class ProjectStatus(str, Enum): + CREATED = "created" + ONTOLOGY_GENERATED = "ontology_generated" + GRAPH_BUILDING = "graph_building" + GRAPH_COMPLETED = "graph_completed" + FAILED = "failed" + + +class SimulationStatus(str, Enum): + CREATED = "created" + PREPARING = "preparing" + READY = "ready" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +class ReportStatus(str, Enum): + DRAFT = "draft" + PLANNING = "planning" + GENERATING = "generating" + COMPLETED = "completed" + FAILED = "failed" + + +class ProductProject(Base): + """Durable, tenant-owned project record (metadata + ontology).""" + + __tablename__ = "projects" + __table_args__ = ( + Index("ix_projects_org_owner", "organization_id", "owner_user_id"), + Index("ix_projects_org_created", "organization_id", "created_at"), + ) + + id: Mapped[str] = mapped_column( + String(128), primary_key=True, default=lambda: _id("project") + ) + organization_id: Mapped[str] = mapped_column( + ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True + ) + owner_user_id: Mapped[str | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True + ) + name: Mapped[str] = mapped_column(String(255), nullable=False, server_default=text("''")) + status: Mapped[str] = mapped_column( + String(32), nullable=False, default=ProjectStatus.CREATED.value, server_default=text("'created'") + ) + language: Mapped[str] = mapped_column( + String(16), nullable=False, default="en", server_default=text("'en'") + ) + total_text_length: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default=text("0") + ) + source_metadata: Mapped[dict | list | None] = mapped_column(JSON, nullable=True) + ontology: Mapped[dict | list | None] = mapped_column(JSON, nullable=True) + analysis_summary: Mapped[str | None] = mapped_column(Text, nullable=True) + simulation_requirement: Mapped[str | None] = mapped_column(Text, nullable=True) + graph_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True) + graph_build_task_id: Mapped[str | None] = mapped_column(String(128), nullable=True) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now(), onupdate=_utc_now + ) + + +class ProductSimulation(Base): + """Durable, tenant-scoped simulation with a config snapshot.""" + + __tablename__ = "simulations" + __table_args__ = ( + Index("ix_simulations_org_project", "organization_id", "project_id"), + Index("ix_simulations_org_created", "organization_id", "created_at"), + ) + + id: Mapped[str] = mapped_column( + String(128), primary_key=True, default=lambda: _id("sim") + ) + organization_id: Mapped[str] = mapped_column( + ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True + ) + project_id: Mapped[str] = mapped_column( + ForeignKey("projects.id", ondelete="CASCADE"), nullable=False, index=True + ) + created_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + status: Mapped[str] = mapped_column( + String(32), nullable=False, default=SimulationStatus.CREATED.value, server_default=text("'created'") + ) + platform: Mapped[str] = mapped_column( + String(32), nullable=False, default="parallel", server_default=text("'parallel'") + ) + config: Mapped[dict | list | None] = mapped_column(JSON, nullable=True) + current_round: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0")) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now(), onupdate=_utc_now + ) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + +class DurableReport(Base): + """Durable, tenant-scoped report with outline/status/content metadata.""" + + __tablename__ = "reports" + __table_args__ = ( + UniqueConstraint("organization_id", "id", name="uq_reports_org_id"), + Index("ix_reports_org_project", "organization_id", "project_id"), + Index("ix_reports_org_simulation", "organization_id", "simulation_id"), + Index("ix_reports_org_created", "organization_id", "created_at"), + ) + + id: Mapped[str] = mapped_column( + String(128), primary_key=True, default=lambda: _id("report") + ) + organization_id: Mapped[str] = mapped_column( + ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True + ) + project_id: Mapped[str] = mapped_column( + ForeignKey("projects.id", ondelete="CASCADE"), nullable=False, index=True + ) + simulation_id: Mapped[str | None] = mapped_column( + ForeignKey("simulations.id", ondelete="SET NULL"), nullable=True, index=True + ) + created_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + status: Mapped[str] = mapped_column( + String(32), nullable=False, default=ReportStatus.DRAFT.value, server_default=text("'draft'") + ) + title: Mapped[str] = mapped_column(String(255), nullable=False, server_default=text("''")) + outline: Mapped[dict | list | None] = mapped_column(JSON, nullable=True) + markdown_content: Mapped[str | None] = mapped_column(Text, nullable=True) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now(), onupdate=_utc_now + ) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) diff --git a/backend/app/models/project.py b/backend/app/models/project.py index 0897893..533cf79 100644 --- a/backend/app/models/project.py +++ b/backend/app/models/project.py @@ -5,8 +5,10 @@ import os import json +import re import uuid import shutil +import tempfile from datetime import datetime from typing import Dict, Any, List, Optional from enum import Enum @@ -31,6 +33,8 @@ class Project: status: ProjectStatus created_at: str updated_at: str + organization_id: Optional[str] = None + owner_user_id: Optional[str] = None # 文件信息 files: List[Dict[str, str]] = field(default_factory=list) # [{filename, path, size}] @@ -60,6 +64,8 @@ class Project: "status": self.status.value if isinstance(self.status, ProjectStatus) else self.status, "created_at": self.created_at, "updated_at": self.updated_at, + "organization_id": self.organization_id, + "owner_user_id": self.owner_user_id, "files": self.files, "total_text_length": self.total_text_length, "ontology": self.ontology, @@ -85,6 +91,8 @@ class Project: status=status, created_at=data.get('created_at', ''), updated_at=data.get('updated_at', ''), + organization_id=data.get('organization_id'), + owner_user_id=data.get('owner_user_id'), files=data.get('files', []), total_text_length=data.get('total_text_length', 0), ontology=data.get('ontology'), @@ -100,6 +108,8 @@ class Project: class ProjectManager: """项目管理器 - 负责项目的持久化存储和检索""" + + _SAFE_PROJECT_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$") # 项目存储根目录 PROJECTS_DIR = os.path.join(Config.UPLOAD_FOLDER, 'projects') @@ -109,10 +119,21 @@ class ProjectManager: """确保项目目录存在""" os.makedirs(cls.PROJECTS_DIR, exist_ok=True) + @classmethod + def _validate_project_id(cls, project_id: str) -> str: + if not isinstance(project_id, str) or not cls._SAFE_PROJECT_ID.fullmatch(project_id): + raise ValueError("invalid_project_id") + return project_id + @classmethod def _get_project_dir(cls, project_id: str) -> str: - """获取项目目录路径""" - return os.path.join(cls.PROJECTS_DIR, project_id) + """获取项目目录路径，拒绝路径分隔符和 traversal。""" + safe_project_id = cls._validate_project_id(project_id) + root = os.path.realpath(cls.PROJECTS_DIR) + project_dir = os.path.realpath(os.path.join(root, safe_project_id)) + if os.path.commonpath([root, project_dir]) != root: + raise ValueError("invalid_project_id") + return project_dir @classmethod def _get_project_meta_path(cls, project_id: str) -> str: @@ -130,7 +151,12 @@ class ProjectManager: return os.path.join(cls._get_project_dir(project_id), 'extracted_text.txt') @classmethod - def create_project(cls, name: str = "Unnamed Project") -> Project: + def create_project( + cls, + name: str = "Unnamed Project", + organization_id: Optional[str] = None, + owner_user_id: Optional[str] = None, + ) -> Project: """ 创建新项目 @@ -150,7 +176,9 @@ class ProjectManager: name=name, status=ProjectStatus.CREATED, created_at=now, - updated_at=now + updated_at=now, + organization_id=organization_id, + owner_user_id=owner_user_id, ) # 创建项目目录结构 @@ -166,12 +194,24 @@ class ProjectManager: @classmethod def save_project(cls, project: Project) -> None: - """保存项目元数据""" + """保存项目元数据，使用同目录临时文件+原子替换。""" project.updated_at = datetime.now().isoformat() + project_dir = cls._get_project_dir(project.project_id) + os.makedirs(project_dir, exist_ok=True) meta_path = cls._get_project_meta_path(project.project_id) - - with open(meta_path, 'w', encoding='utf-8') as f: - json.dump(project.to_dict(), f, ensure_ascii=False, indent=2) + fd, temp_path = tempfile.mkstemp(prefix=".project-", suffix=".json", dir=project_dir) + try: + with os.fdopen(fd, 'w', encoding='utf-8') as f: + json.dump(project.to_dict(), f, ensure_ascii=False, indent=2) + f.flush() + os.fsync(f.fileno()) + os.replace(temp_path, meta_path) + except Exception: + try: + os.unlink(temp_path) + except FileNotFoundError: + pass + raise @classmethod def get_project(cls, project_id: str) -> Optional[Project]: @@ -184,18 +224,32 @@ class ProjectManager: Returns: Project对象，如果不存在返回None """ - meta_path = cls._get_project_meta_path(project_id) - + try: + meta_path = cls._get_project_meta_path(project_id) + except ValueError: + return None + if not os.path.exists(meta_path): return None - with open(meta_path, 'r', encoding='utf-8') as f: - data = json.load(f) + try: + with open(meta_path, 'r', encoding='utf-8') as f: + data = json.load(f) + except (OSError, json.JSONDecodeError, TypeError, KeyError, ValueError): + return None - return Project.from_dict(data) + try: + return Project.from_dict(data) + except (KeyError, TypeError, ValueError): + return None @classmethod - def list_projects(cls, limit: int = 50) -> List[Project]: + def list_projects( + cls, + limit: int = 50, + organization_id: Optional[str] = None, + owner_user_id: Optional[str] = None, + ) -> List[Project]: """ 列出所有项目 @@ -210,30 +264,84 @@ class ProjectManager: projects = [] for project_id in os.listdir(cls.PROJECTS_DIR): project = cls.get_project(project_id) - if project: - projects.append(project) + if not project: + continue + if organization_id is not None and project.organization_id != organization_id: + continue + if owner_user_id is not None and project.owner_user_id != owner_user_id: + continue + projects.append(project) # 按创建时间倒序排序 projects.sort(key=lambda p: p.created_at, reverse=True) return projects[:limit] + + @classmethod + def find_project_by_graph_id( + cls, + graph_id: str, + *, + organization_id: str, + owner_user_id: Optional[str] = None, + ) -> Optional[Project]: + for project in cls.list_projects( + organization_id=organization_id, + owner_user_id=owner_user_id, + ): + if project.graph_id == graph_id: + return project + return None + + @classmethod + def get_project_for_scope( + cls, + project_id: str, + *, + organization_id: str, + owner_user_id: Optional[str] = None, + ) -> Optional[Project]: + """Return only an explicitly owned/scoped project; legacy records fail closed.""" + if not isinstance(organization_id, str) or not organization_id: + return None + project = cls.get_project(project_id) + if project is None or project.organization_id != organization_id: + return None + if owner_user_id is not None and project.owner_user_id != owner_user_id: + return None + return project @classmethod - def delete_project(cls, project_id: str) -> bool: + def delete_project( + cls, + project_id: str, + *, + organization_id: str, + owner_user_id: Optional[str] = None, + ) -> bool: """ - 删除项目及其所有文件 - + 删除项目及其所有文件，但只允许删除明确授权范围内的项目。 + Args: project_id: 项目ID - + organization_id: 当前请求的组织范围 + owner_user_id: 普通用户的所有者范围；管理员可留空 + Returns: 是否删除成功 """ - project_dir = cls._get_project_dir(project_id) - + project = cls.get_project_for_scope( + project_id, + organization_id=organization_id, + owner_user_id=owner_user_id, + ) + if project is None: + return False + + project_dir = cls._get_project_dir(project.project_id) if not os.path.exists(project_dir): return False - + shutil.rmtree(project_dir) return True diff --git a/backend/app/models/rate_limit.py b/backend/app/models/rate_limit.py new file mode 100644 index 0000000..4629aec --- /dev/null +++ b/backend/app/models/rate_limit.py @@ -0,0 +1,35 @@ +"""Durable rate-limit event records.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import uuid4 + +from sqlalchemy import DateTime, Index, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from ..db import Base + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class RateLimitEvent(Base): + """One recorded rate-limit hit for an operation + key (no secrets).""" + + __tablename__ = "rate_limit_events" + __table_args__ = ( + Index("ix_rate_limit_op_key_created", "operation", "key", "created_at"), + Index("ix_rate_limit_org_created", "organization_id", "created_at"), + ) + + id: Mapped[str] = mapped_column( + String(64), primary_key=True, default=lambda: f"rl_{uuid4().hex}" + ) + operation: Mapped[str] = mapped_column(String(120), nullable=False) + key: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + organization_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now() + ) diff --git a/backend/app/models/saas.py b/backend/app/models/saas.py new file mode 100644 index 0000000..186b027 --- /dev/null +++ b/backend/app/models/saas.py @@ -0,0 +1,138 @@ +"""SQLAlchemy identity and tenant metadata models.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import List +from uuid import uuid4 + +from sqlalchemy import CheckConstraint, DateTime, Enum as SAEnum, ForeignKey, Integer, String, UniqueConstraint, func, text +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from ..db import Base +from ..security.policy import Role + + +def _id(prefix: str) -> str: + return f"{prefix}_{uuid4().hex}" + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _role_values(enum_type): + return [member.value for member in enum_type] + + +class Organization(Base): + __tablename__ = "organizations" + + id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: _id("org")) + name: Mapped[str] = mapped_column(String(160), nullable=False) + slug: Mapped[str] = mapped_column(String(80), nullable=False, unique=True, index=True) + status: Mapped[str] = mapped_column( + String(32), nullable=False, default="active", server_default=text("'active'") + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utc_now, server_default=func.now(), nullable=False + ) + + memberships: Mapped[List["Membership"]] = relationship( + back_populates="organization", cascade="all, delete-orphan" + ) + + +class User(Base): + __tablename__ = "users" + + id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: _id("usr")) + email_normalized: Mapped[str] = mapped_column(String(320), nullable=False, unique=True, index=True) + password_hash: Mapped[str] = mapped_column( + String(512), nullable=False, default="!invite_pending", server_default=text("'!invite_pending'") + ) + status: Mapped[str] = mapped_column( + String(32), nullable=False, default="active", server_default=text("'active'") + ) + auth_version: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default=text("0") + ) + locale: Mapped[str] = mapped_column( + String(8), nullable=False, default="th", server_default=text("'th'") + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utc_now, server_default=func.now(), nullable=False + ) + last_login_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + memberships: Mapped[List["Membership"]] = relationship( + back_populates="user", cascade="all, delete-orphan" + ) + sessions: Mapped[List["AuthSession"]] = relationship( + back_populates="user", cascade="all, delete-orphan" + ) + + +class Membership(Base): + __tablename__ = "memberships" + __table_args__ = ( + UniqueConstraint("user_id", "organization_id", name="uq_membership_user_org"), + CheckConstraint( + "role IN ('super_admin', 'admin', 'user')", + name="ck_membership_role", + ), + ) + + id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: _id("mem")) + user_id: Mapped[str] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + organization_id: Mapped[str] = mapped_column( + ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True + ) + role: Mapped[Role] = mapped_column( + SAEnum( + Role, + name="role", + values_callable=_role_values, + native_enum=False, + create_constraint=False, + validate_strings=True, + ), + nullable=False, + ) + status: Mapped[str] = mapped_column( + String(32), nullable=False, default="active", server_default=text("'active'") + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utc_now, server_default=func.now(), nullable=False + ) + + user: Mapped[User] = relationship(back_populates="memberships") + organization: Mapped[Organization] = relationship(back_populates="memberships") + sessions: Mapped[List["AuthSession"]] = relationship(back_populates="membership") + + +class AuthSession(Base): + __tablename__ = "sessions" + + id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: _id("ses")) + user_id: Mapped[str] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + membership_id: Mapped[str] = mapped_column( + ForeignKey("memberships.id", ondelete="CASCADE"), nullable=False, index=True + ) + token_hash: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True) + auth_version: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default=text("0") + ) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utc_now, server_default=func.now(), nullable=False + ) + last_seen_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + user: Mapped[User] = relationship(back_populates="sessions") + membership: Mapped[Membership] = relationship(back_populates="sessions") diff --git a/backend/app/models/settings.py b/backend/app/models/settings.py new file mode 100644 index 0000000..fd8182b --- /dev/null +++ b/backend/app/models/settings.py @@ -0,0 +1,42 @@ +"""Durable, versioned platform settings (LLM provider etc.) with redacted secrets.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import uuid4 + +from sqlalchemy import Boolean, DateTime, ForeignKey, JSON, String, func, text +from sqlalchemy.orm import Mapped, mapped_column + +from ..db import Base + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class PlatformSettings(Base): + """A versioned snapshot of platform LLM settings. + + Public/non-secret settings live in ``settings`` (JSON). The API key must be + stored encrypted (as ``secret_ref``), never as plaintext in ``settings``. + ``active`` marks the current effective version. + """ + + __tablename__ = "platform_settings" + + id: Mapped[str] = mapped_column( + String(64), primary_key=True, default=lambda: f"ps_{uuid4().hex}" + ) + version: Mapped[str] = mapped_column(String(64), nullable=False, unique=True) + settings: Mapped[dict | None] = mapped_column(JSON, nullable=True) + secret_ref: Mapped[str | None] = mapped_column(String(512), nullable=True) + updated_by_user_id: Mapped[str | None] = mapped_column( + String(64), nullable=True + ) + active: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False, server_default=text("0") + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now() + ) diff --git a/backend/app/models/task.py b/backend/app/models/task.py index dfebed2..86f44c9 100644 --- a/backend/app/models/task.py +++ b/backend/app/models/task.py @@ -1,43 +1,48 @@ -""" -任务状态管理 -用于跟踪长时间运行的任务（如图谱构建） +"""Durable task status management with a test-only in-memory fallback. + +The Flask application configures a SQLAlchemy session factory at startup. Code +that uses TaskManager outside an application (small unit tests and legacy +adapters) keeps the old in-memory behavior, but production requests do not. """ -import uuid +from __future__ import annotations + import threading -from datetime import datetime -from enum import Enum -from typing import Dict, Any, Optional +import uuid from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from enum import Enum +from typing import Any, Dict, Optional, cast +from flask import current_app, has_app_context +from sqlalchemy import delete, select + +from ..models.operations import Job, JobStatus from ..utils.locale import t class TaskStatus(str, Enum): - """任务状态枚举""" - PENDING = "pending" # 等待中 - PROCESSING = "processing" # 处理中 - COMPLETED = "completed" # 已完成 - FAILED = "failed" # 失败 + PENDING = "pending" + PROCESSING = "processing" + COMPLETED = "completed" + FAILED = "failed" @dataclass class Task: - """任务数据类""" task_id: str task_type: str status: TaskStatus created_at: datetime updated_at: datetime - progress: int = 0 # 总进度百分比 0-100 - message: str = "" # 状态消息 - result: Optional[Dict] = None # 任务结果 - error: Optional[str] = None # 错误信息 - metadata: Dict = field(default_factory=dict) # 额外元数据 - progress_detail: Dict = field(default_factory=dict) # 详细进度信息 - + progress: int = 0 + message: str = "" + result: Optional[Dict] = None + error: Optional[str] = None + metadata: Dict = field(default_factory=dict) + progress_detail: Dict = field(default_factory=dict) + def to_dict(self) -> Dict[str, Any]: - """转换为字典""" return { "task_id": self.task_id, "task_type": self.task_type, @@ -54,57 +59,182 @@ class Task: class TaskManager: - """ - 任务管理器 - 线程安全的任务状态管理 - """ - - _instance = None - _lock = threading.Lock() - - def __new__(cls): - """单例模式""" - if cls._instance is None: - with cls._lock: - if cls._instance is None: - cls._instance = super().__new__(cls) - cls._instance._tasks: Dict[str, Task] = {} - cls._instance._task_lock = threading.Lock() - return cls._instance - + """Thread-safe task facade bound to one app/session factory.""" + + _configured_session_factory = None + _config_lock = threading.Lock() + _fallback_tasks: Dict[str, Task] = {} + _fallback_lock = threading.Lock() + + def __init__(self, session_factory=None): + """Bind this manager to an explicit or current-app session factory. + + A manager created during a request/app context keeps that app's factory + for background work, but it cannot be reused inside a different Flask + app. The explicit class configuration remains only for legacy tests and + callers that run outside Flask. + """ + self._bound_app = None + if has_app_context(): + app = cast(Any, current_app)._get_current_object() + current_factory = app.extensions.get("crowdsight_session_factory") + if not callable(current_factory): + raise RuntimeError("task_session_factory_required") + if session_factory is not None and session_factory is not current_factory: + raise RuntimeError("task_session_factory_mismatch") + bound_factory = current_factory + self._bound_app = app + elif session_factory is not None: + bound_factory = session_factory + else: + # No Flask app context: use the explicit legacy/test binding. + bound_factory = type(self)._configured_session_factory + + self._session_factory = bound_factory + self._tasks = type(self)._fallback_tasks + self._task_lock = type(self)._fallback_lock + + @classmethod + def configure(cls, session_factory) -> None: + """Set an explicit outside-Flask binding for tests/legacy adapters.""" + with cls._config_lock: + cls._configured_session_factory = session_factory + with cls._fallback_lock: + cls._fallback_tasks.clear() + + def _factory(self): + if not has_app_context(): + return self._session_factory + + app = cast(Any, current_app)._get_current_object() + current_factory = app.extensions.get("crowdsight_session_factory") + if not callable(current_factory): + raise RuntimeError("task_session_factory_required") + if self._bound_app is not None and self._bound_app is not app: + raise RuntimeError("task_app_context_mismatch") + if self._session_factory is not None and self._session_factory is not current_factory: + raise RuntimeError("task_session_factory_mismatch") + if self._session_factory is None: + self._session_factory = current_factory + self._bound_app = app + return self._session_factory + + @staticmethod + def _bounded_text(value: Any, limit: int = 4000) -> str: + if value is None: + return "" + return str(value)[:limit] + + @staticmethod + def _job_status(status: TaskStatus | str | None) -> str | None: + if status is None: + return None + value = status.value if isinstance(status, TaskStatus) else str(status) + return { + TaskStatus.PENDING.value: JobStatus.QUEUED.value, + TaskStatus.PROCESSING.value: JobStatus.RUNNING.value, + TaskStatus.COMPLETED.value: JobStatus.SUCCEEDED.value, + TaskStatus.FAILED.value: JobStatus.FAILED.value, + JobStatus.QUEUED.value: JobStatus.QUEUED.value, + JobStatus.RUNNING.value: JobStatus.RUNNING.value, + JobStatus.SUCCEEDED.value: JobStatus.SUCCEEDED.value, + JobStatus.FAILED.value: JobStatus.FAILED.value, + JobStatus.CANCELLED.value: JobStatus.CANCELLED.value, + }.get(value) + + @staticmethod + def _task_status(status: str | JobStatus) -> TaskStatus: + value = status.value if isinstance(status, JobStatus) else str(status) + return { + JobStatus.QUEUED.value: TaskStatus.PENDING, + JobStatus.RUNNING.value: TaskStatus.PROCESSING, + JobStatus.SUCCEEDED.value: TaskStatus.COMPLETED, + JobStatus.FAILED.value: TaskStatus.FAILED, + JobStatus.CANCELLED.value: TaskStatus.FAILED, + }.get(value, TaskStatus.FAILED) + + @classmethod + def _from_job(cls, job: Job) -> Task: + metadata = job.job_metadata if isinstance(job.job_metadata, dict) else {} + result = job.result if isinstance(job.result, dict) else job.result + detail = job.progress_detail if isinstance(job.progress_detail, dict) else {} + return Task( + task_id=job.id, + task_type=job.operation, + status=cls._task_status(job.status), + created_at=job.created_at, + updated_at=job.updated_at, + progress=job.progress, + message=job.message, + result=result, + error=job.error_code, + metadata=metadata, + progress_detail=detail, + ) + def create_task(self, task_type: str, metadata: Optional[Dict] = None) -> str: - """ - 创建新任务 - - Args: - task_type: 任务类型 - metadata: 额外元数据 - - Returns: - 任务ID - """ + metadata = metadata or {} + factory = self._factory() + if factory is not None: + organization_id = metadata.get("organization_id") + if not isinstance(organization_id, str) or not organization_id: + raise ValueError("task_scope_required") + with factory() as session: + job = Job( + organization_id=organization_id, + owner_user_id=metadata.get("owner_user_id"), + project_id=metadata.get("project_id"), + graph_id=metadata.get("graph_id"), + operation=self._bounded_text(task_type, 120), + status=JobStatus.QUEUED.value, + job_metadata=metadata, + progress_detail={}, + ) + session.add(job) + session.commit() + return job.id + task_id = str(uuid.uuid4()) - now = datetime.now() - + now = datetime.now(timezone.utc) task = Task( task_id=task_id, task_type=task_type, status=TaskStatus.PENDING, created_at=now, updated_at=now, - metadata=metadata or {} + metadata=metadata, ) - with self._task_lock: self._tasks[task_id] = task - return task_id - - def get_task(self, task_id: str) -> Optional[Task]: - """获取任务""" + + def get_task( + self, + task_id: str, + *, + organization_id: Optional[str] = None, + owner_user_id: Optional[str] = None, + ) -> Optional[Task]: + factory = self._factory() + if factory is not None: + with factory() as session: + statement = select(Job).where(Job.id == task_id) + if organization_id is not None: + statement = statement.where(Job.organization_id == organization_id) + if owner_user_id is not None: + statement = statement.where(Job.owner_user_id == owner_user_id) + job = session.scalar(statement) + return self._from_job(job) if job is not None else None with self._task_lock: - return self._tasks.get(task_id) - + task = self._tasks.get(task_id) + if task is None: + return None + if organization_id is not None and task.metadata.get("organization_id") != organization_id: + return None + if owner_user_id is not None and task.metadata.get("owner_user_id") != owner_user_id: + return None + return task + def update_task( self, task_id: str, @@ -113,28 +243,45 @@ class TaskManager: message: Optional[str] = None, result: Optional[Dict] = None, error: Optional[str] = None, - progress_detail: Optional[Dict] = None + progress_detail: Optional[Dict] = None, ): - """ - 更新任务状态 - - Args: - task_id: 任务ID - status: 新状态 - progress: 进度 - message: 消息 - result: 结果 - error: 错误信息 - progress_detail: 详细进度信息 - """ + factory = self._factory() + if factory is not None: + with factory() as session: + job = session.get(Job, task_id) + if job is None: + return + mapped_status = self._job_status(status) + if mapped_status is not None: + job.status = mapped_status + if mapped_status in { + JobStatus.SUCCEEDED.value, + JobStatus.FAILED.value, + JobStatus.CANCELLED.value, + }: + job.finished_at = datetime.now(timezone.utc) + if progress is not None: + job.progress = min(max(int(progress), 0), 100) + if message is not None: + job.message = self._bounded_text(message) + if result is not None: + job.result = result + if error is not None: + job.error_code = self._bounded_text(error, 120) + if progress_detail is not None: + job.progress_detail = progress_detail + job.updated_at = datetime.now(timezone.utc) + session.commit() + return + with self._task_lock: task = self._tasks.get(task_id) if task: - task.updated_at = datetime.now() + task.updated_at = datetime.now(timezone.utc) if status is not None: task.status = status if progress is not None: - task.progress = progress + task.progress = min(max(int(progress), 0), 100) if message is not None: task.message = message if result is not None: @@ -143,44 +290,71 @@ class TaskManager: task.error = error if progress_detail is not None: task.progress_detail = progress_detail - + def complete_task(self, task_id: str, result: Dict): - """标记任务完成""" self.update_task( task_id, status=TaskStatus.COMPLETED, progress=100, - message=t('progress.taskComplete'), - result=result + message=t("progress.taskComplete"), + result=result, ) - + def fail_task(self, task_id: str, error: str): - """标记任务失败""" self.update_task( task_id, status=TaskStatus.FAILED, - message=t('progress.taskFailed'), - error=error + message=t("progress.taskFailed"), + error=error, ) - - def list_tasks(self, task_type: Optional[str] = None) -> list: - """列出任务""" + + def list_tasks( + self, + task_type: Optional[str] = None, + *, + organization_id: Optional[str] = None, + owner_user_id: Optional[str] = None, + ) -> list: + factory = self._factory() + if factory is not None: + with factory() as session: + statement = select(Job).order_by(Job.created_at.desc()) + if task_type: + statement = statement.where(Job.operation == task_type) + if organization_id is not None: + statement = statement.where(Job.organization_id == organization_id) + if owner_user_id is not None: + statement = statement.where(Job.owner_user_id == owner_user_id) + jobs = session.scalars(statement.limit(100)).all() + return [self._from_job(job) for job in jobs] with self._task_lock: tasks = list(self._tasks.values()) if task_type: - tasks = [t for t in tasks if t.task_type == task_type] - return [t.to_dict() for t in sorted(tasks, key=lambda x: x.created_at, reverse=True)] - + tasks = [task for task in tasks if task.task_type == task_type] + if organization_id is not None: + tasks = [task for task in tasks if task.metadata.get("organization_id") == organization_id] + if owner_user_id is not None: + tasks = [task for task in tasks if task.metadata.get("owner_user_id") == owner_user_id] + return [task for task in sorted(tasks, key=lambda item: item.created_at, reverse=True)] + def cleanup_old_tasks(self, max_age_hours: int = 24): - """清理旧任务""" - from datetime import timedelta - cutoff = datetime.now() - timedelta(hours=max_age_hours) - + cutoff = datetime.now(timezone.utc) - timedelta(hours=max_age_hours) + factory = self._factory() + if factory is not None: + with factory() as session: + session.execute( + delete(Job).where( + Job.created_at < cutoff, + Job.status.in_([JobStatus.SUCCEEDED.value, JobStatus.FAILED.value]), + ) + ) + session.commit() + return with self._task_lock: old_ids = [ - tid for tid, task in self._tasks.items() + task_id + for task_id, task in self._tasks.items() if task.created_at < cutoff and task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED] ] - for tid in old_ids: - del self._tasks[tid] - + for task_id in old_ids: + del self._tasks[task_id] diff --git a/backend/app/models/usage.py b/backend/app/models/usage.py new file mode 100644 index 0000000..e4ffc99 --- /dev/null +++ b/backend/app/models/usage.py @@ -0,0 +1,45 @@ +"""Durable LLM usage/cost events.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import uuid4 + +from sqlalchemy import DateTime, Float, ForeignKey, Index, Integer, String, func, text +from sqlalchemy.orm import Mapped, mapped_column + +from ..db import Base + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class UsageEvent(Base): + """One LLM usage record. Never stores prompt content or secrets.""" + + __tablename__ = "usage_events" + __table_args__ = ( + Index("ix_usage_org_created", "organization_id", "created_at"), + Index("ix_usage_org_user", "organization_id", "user_id"), + ) + + id: Mapped[str] = mapped_column( + String(64), primary_key=True, default=lambda: f"usage_{uuid4().hex}" + ) + organization_id: Mapped[str] = mapped_column( + ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True + ) + user_id: Mapped[str | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True + ) + operation: Mapped[str] = mapped_column(String(160), nullable=False) + model: Mapped[str | None] = mapped_column(String(120), nullable=True) + input_tokens: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0")) + output_tokens: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0")) + estimated_cost: Mapped[float] = mapped_column( + Float, nullable=False, default=0.0, server_default=text("0") + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_utc_now, server_default=func.now() + ) diff --git a/backend/app/security/__init__.py b/backend/app/security/__init__.py new file mode 100644 index 0000000..8490d4e --- /dev/null +++ b/backend/app/security/__init__.py @@ -0,0 +1,19 @@ +"""Security policy package.""" + +from .policy import ( + Actor, + AuthorizationError, + Role, + assert_can_access_resource, + assert_can_manage_llm_settings, + assert_can_manage_user, +) + +__all__ = [ + "Actor", + "AuthorizationError", + "Role", + "assert_can_access_resource", + "assert_can_manage_llm_settings", + "assert_can_manage_user", +] diff --git a/backend/app/security/auth.py b/backend/app/security/auth.py new file mode 100644 index 0000000..894401e --- /dev/null +++ b/backend/app/security/auth.py @@ -0,0 +1,108 @@ +"""Flask request authentication decorators for tenant-scoped routes.""" + +from __future__ import annotations + +import hmac +import secrets +from functools import wraps + +from flask import current_app, g, request +from itsdangerous import BadSignature, URLSafeTimedSerializer + +from ..services.identity import SessionService +from ..utils.api_errors import ApiError +from .policy import Actor, Role + + +def _session_factory(): + factory = current_app.extensions.get("crowdsight_session_factory") + if factory is None: + raise ApiError("auth_unavailable", 503, "api.internalError") + return factory + + +def _csrf_serializer() -> URLSafeTimedSerializer: + secret_key = current_app.secret_key + if not secret_key: + raise ApiError("auth_unavailable", 503, "api.internalError") + return URLSafeTimedSerializer(secret_key, salt="crowdsight-csrf") + + +def issue_csrf_token() -> str: + return _csrf_serializer().dumps(secrets.token_urlsafe(24)) + + +def _validate_csrf() -> None: + if request.method in {"GET", "HEAD", "OPTIONS"}: + return + cookie_token = request.cookies.get("crowdsight_csrf", "") + header_token = request.headers.get("X-CSRF-Token", "") + if not cookie_token or not header_token or not hmac.compare_digest(cookie_token, header_token): + raise ApiError("csrf_failed", 403, "common.error") + try: + _csrf_serializer().loads(cookie_token, max_age=SessionService.DEFAULT_TTL_SECONDS) + except BadSignature as exc: + raise ApiError("csrf_failed", 403, "common.error") from exc + + +def authenticate_readonly_request() -> None: + """Authenticate a legacy blueprint without exposing a DB session to the route.""" + if getattr(g, "auth_context", None) is not None: + return + raw_token = request.cookies.get("crowdsight_session", "") + with _session_factory()() as db_session: + context = SessionService.resolve(db_session, raw_token) + if context is None: + raise ApiError("unauthorized", 401, "common.unauthorized") + _validate_csrf() + g.auth_context = context + + +def current_actor() -> Actor: + context = getattr(g, "auth_context", None) + if context is None: + raise ApiError("unauthorized", 401, "common.unauthorized") + return Actor( + user_id=context.user.id, + organization_id=context.organization.id, + role=context.membership.role, + ) + + +def require_auth(view): + @wraps(view) + def wrapped(*args, **kwargs): + raw_token = request.cookies.get("crowdsight_session", "") + with _session_factory()() as db_session: + context = SessionService.resolve(db_session, raw_token) + if context is None: + raise ApiError("unauthorized", 401, "common.unauthorized") + _validate_csrf() + + g.auth_context = context + g.db_session = db_session + try: + response = view(*args, **kwargs) + db_session.commit() + return response + except Exception: + db_session.rollback() + raise + + return wrapped + + +def require_roles(*allowed_roles: Role | str): + allowed = {role if isinstance(role, Role) else Role(role) for role in allowed_roles} + + def decorator(view): + @wraps(view) + def wrapped(*args, **kwargs): + actor = current_actor() + if actor.role not in allowed: + raise ApiError("forbidden", 403, "common.error") + return view(*args, **kwargs) + + return require_auth(wrapped) + + return decorator diff --git a/backend/app/security/policy.py b/backend/app/security/policy.py new file mode 100644 index 0000000..868e0cd --- /dev/null +++ b/backend/app/security/policy.py @@ -0,0 +1,121 @@ +"""Fail-closed authorization primitives for tenant-scoped resource services. + +This module deliberately has no Flask or database dependency. Route handlers and +repositories can use the same policy contract without importing the whole app. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any + + +class Role(str, Enum): + SUPER_ADMIN = "super_admin" + ADMIN = "admin" + USER = "user" + + +class AuthorizationError(PermissionError): + """Raised when an actor cannot perform a requested operation.""" + + def __init__(self, code: str = "forbidden"): + self.code = code + super().__init__(code) + + +@dataclass(frozen=True) +class Actor: + """Minimal authenticated identity required by policy checks.""" + + user_id: str + organization_id: str + role: Role | str + + +def _role(actor: Actor) -> Role: + try: + return actor.role if isinstance(actor.role, Role) else Role(actor.role) + except (TypeError, ValueError) as exc: + raise AuthorizationError("invalid_actor_role") from exc + + +def _require_non_empty(value: Any, code: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise AuthorizationError(code) + return value + + +def assert_can_access_resource( + actor: Actor, + *, + resource_organization_id: str, + owner_user_id: str | None, + action: str = "read", + platform_scope: bool = False, +) -> None: + """Raise unless ``actor`` may access a tenant-owned resource. + + ``platform_scope`` is explicit even for super admins so a caller cannot + accidentally turn every ordinary resource lookup into a cross-tenant path. + A regular user must have an exact owner marker; missing/falsey ownership is + denied rather than treated as public. + """ + del action # The first contract is scope; action-specific rules layer on it. + role = _role(actor) + resource_org = _require_non_empty(resource_organization_id, "invalid_resource_scope") + actor_org = _require_non_empty(actor.organization_id, "invalid_actor_scope") + + if role is Role.SUPER_ADMIN: + if platform_scope or actor_org == resource_org: + return + raise AuthorizationError("cross_tenant_scope_required") + + if actor_org != resource_org: + raise AuthorizationError("cross_tenant_forbidden") + + if role is Role.ADMIN: + return + + if role is Role.USER and owner_user_id == actor.user_id and actor.user_id: + return + + raise AuthorizationError("resource_owner_required") + + +def assert_can_manage_user( + actor: Actor, + *, + target_organization_id: str, + target_role: Role | str, + platform_scope: bool = False, +) -> None: + """Raise unless ``actor`` may manage a target account/membership.""" + role = _role(actor) + target_org = _require_non_empty(target_organization_id, "invalid_target_scope") + try: + requested_role = target_role if isinstance(target_role, Role) else Role(target_role) + except (TypeError, ValueError) as exc: + raise AuthorizationError("invalid_target_role") from exc + + if role is Role.SUPER_ADMIN: + if platform_scope or actor.organization_id == target_org: + return + raise AuthorizationError("cross_tenant_scope_required") + + if role is Role.ADMIN: + if actor.organization_id != target_org: + raise AuthorizationError("cross_tenant_forbidden") + if requested_role is Role.USER: + return + raise AuthorizationError("admin_role_grant_forbidden") + + raise AuthorizationError("user_management_forbidden") + + +def assert_can_manage_llm_settings(actor: Actor, *, platform_scope: bool = False) -> None: + """Only a super admin with explicit platform scope may mutate LLM settings.""" + if _role(actor) is Role.SUPER_ADMIN and platform_scope: + return + raise AuthorizationError("llm_settings_forbidden") diff --git a/backend/app/security/resources.py b/backend/app/security/resources.py new file mode 100644 index 0000000..0089788 --- /dev/null +++ b/backend/app/security/resources.py @@ -0,0 +1,223 @@ +"""Fail-closed tenant/owner lookup helpers for file-backed resources with +durable (PostgreSQL-target) read-first cutover for projects. + +Project lookups prefer the durable ``projects`` table when the session factory +is available, then fall back to the legacy filesystem ``ProjectManager``. This +keeps read paths working during the filesystem → durable migration without +breaking existing routes that consume the legacy ``Project`` dataclass shape. +""" + +from __future__ import annotations + +import re +from typing import Optional + +from flask import current_app, request + +from ..models.project import Project, ProjectManager, ProjectStatus +from ..models.task import Task, TaskManager +from ..services.report_agent import Report, ReportManager +from ..services.simulation_manager import SimulationManager, SimulationState +from ..utils.api_errors import ApiError +from .auth import current_actor +from .policy import Role + +_SAFE_RESOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$") + + +def _valid_resource_id(value: str) -> bool: + return isinstance(value, str) and bool(_SAFE_RESOURCE_ID.fullmatch(value)) + + +def _durable_project_for_scope( + project_id: str, *, organization_id: str, owner_user_id: Optional[str] +) -> Optional[Project]: + """Resolve a project from the durable table and wrap it in the legacy shape. + + Only active when the local (durable) memory backend is selected; the legacy + Zep backend keeps reading the filesystem manager exclusively. Returns None + when the durable store is unavailable or no matching tenant/owner row + exists. Does not raise. + """ + try: + from ..config import Config + + if Config.MEMORY_BACKEND != "local": + return None + session_factory = current_app.extensions.get("crowdsight_session_factory") + if session_factory is None: + return None + from ..services.product_repository import ProductRepository + + session = session_factory() + try: + row = ProductRepository(session).get_project( + project_id, organization_id=organization_id + ) + finally: + session.close() + if row is None: + return None + if owner_user_id is not None and row.owner_user_id != owner_user_id: + return None + try: + project_status = ProjectStatus(row.status) + except ValueError: + project_status = ProjectStatus.CREATED + return Project( + project_id=row.id, + name=row.name, + status=project_status, + created_at=row.created_at.isoformat() if row.created_at else "", + updated_at=row.updated_at.isoformat() if row.updated_at else "", + organization_id=row.organization_id, + owner_user_id=row.owner_user_id, + files=[], + total_text_length=row.total_text_length, + ontology=row.ontology or None, + analysis_summary=row.analysis_summary, + graph_id=row.graph_id, + graph_build_task_id=row.graph_build_task_id, + simulation_requirement=row.simulation_requirement, + error=row.error, + ) + except Exception: + return None + + +def scoped_project(project_id: str) -> Optional[Project]: + if not _valid_resource_id(project_id): + return None + actor = current_actor() + owner_user_id = actor.user_id if actor.role is Role.USER else None + durable = _durable_project_for_scope( + project_id, + organization_id=actor.organization_id, + owner_user_id=owner_user_id, + ) + if durable is not None: + return durable + return ProjectManager.get_project_for_scope( + project_id, + organization_id=actor.organization_id, + owner_user_id=owner_user_id, + ) + + +def require_scoped_project(project_id: str) -> Project: + project = scoped_project(project_id) + if project is None: + raise ApiError("resource_not_found", 404, "common.notFound") + return project + + +def scoped_graph(graph_id: str) -> Optional[Project]: + if not _valid_resource_id(graph_id): + return None + actor = current_actor() + return ProjectManager.find_project_by_graph_id( + graph_id, + organization_id=actor.organization_id, + owner_user_id=actor.user_id if actor.role is Role.USER else None, + ) + + +def scoped_simulation(simulation_id: str) -> Optional[SimulationState]: + if not _valid_resource_id(simulation_id): + return None + state = SimulationManager().get_simulation(simulation_id) + if state is None: + return None + if scoped_project(state.project_id) is None: + return None + return state + + +def require_scoped_simulation(simulation_id: str) -> SimulationState: + state = scoped_simulation(simulation_id) + if state is None: + raise ApiError("resource_not_found", 404, "common.notFound") + return state + + +def scoped_task(task_id: str) -> Optional[Task]: + if not _valid_resource_id(task_id): + return None + actor = current_actor() + task = TaskManager().get_task( + task_id, + organization_id=actor.organization_id, + owner_user_id=actor.user_id if actor.role is Role.USER else None, + ) + if task is None: + return None + metadata = task.metadata if isinstance(task.metadata, dict) else {} + if metadata.get("organization_id") != actor.organization_id: + return None + if actor.role is Role.USER and metadata.get("owner_user_id") != actor.user_id: + return None + return task + + +def require_scoped_task(task_id: str) -> Task: + task = scoped_task(task_id) + if task is None: + raise ApiError("resource_not_found", 404, "common.notFound") + return task + + +def scoped_report(report_id: str) -> Optional[Report]: + if not _valid_resource_id(report_id): + return None + report = ReportManager.get_report(report_id) + if report is None: + return None + if scoped_simulation(report.simulation_id) is None: + return None + return report + + +def require_scoped_report(report_id: str) -> Report: + report = scoped_report(report_id) + if report is None: + raise ApiError("resource_not_found", 404, "common.notFound") + return report + + +def scoped_reports(simulation_id: str | None = None, limit: int = 50) -> list[Report]: + if simulation_id is not None and scoped_simulation(simulation_id) is None: + return [] + safe_limit = min(max(int(limit), 1), 100) + reports = ReportManager.list_reports(simulation_id=simulation_id, limit=safe_limit * 4) + return [report for report in reports if scoped_report(report.report_id) is not None][:safe_limit] + + +def scoped_simulations(project_id: str | None = None) -> list[SimulationState]: + if project_id is not None and scoped_project(project_id) is None: + return [] + simulations = SimulationManager().list_simulations(project_id=project_id) + return [state for state in simulations if scoped_simulation(state.simulation_id) is not None] + + +def enforce_request_scope() -> None: + """Reject IDs outside the current actor's tenant/owner scope before handlers run.""" + view_args = request.view_args or {} + payload = request.get_json(silent=True) + payload = payload if isinstance(payload, dict) else {} + + project_id = view_args.get("project_id") or request.args.get("project_id") or payload.get("project_id") + simulation_id = view_args.get("simulation_id") or request.args.get("simulation_id") or payload.get("simulation_id") + report_id = view_args.get("report_id") or request.args.get("report_id") or payload.get("report_id") + graph_id = view_args.get("graph_id") or request.args.get("graph_id") or payload.get("graph_id") + task_id = view_args.get("task_id") or request.args.get("task_id") or payload.get("task_id") + + if project_id and scoped_project(project_id) is None: + raise ApiError("resource_not_found", 404, "common.notFound") + if simulation_id and scoped_simulation(simulation_id) is None: + raise ApiError("resource_not_found", 404, "common.notFound") + if report_id and scoped_report(report_id) is None: + raise ApiError("resource_not_found", 404, "common.notFound") + if graph_id and scoped_graph(graph_id) is None: + raise ApiError("resource_not_found", 404, "common.notFound") + if task_id and scoped_task(task_id) is None: + raise ApiError("resource_not_found", 404, "common.notFound") diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py index 8db85d8..d7a629c 100644 --- a/backend/app/services/__init__.py +++ b/backend/app/services/__init__.py @@ -1,73 +1,61 @@ -""" -业务服务模块 +"""Business service exports with lazy imports for backend isolation. + +The local memory backend must be importable without eagerly loading Zep-only +consumers. Legacy package-level names remain available through ``__getattr__`` +when a caller explicitly requests them. """ -from .ontology_generator import OntologyGenerator -from .graph_builder import GraphBuilderService -from .text_processor import TextProcessor -from .zep_entity_reader import ZepEntityReader, EntityNode, FilteredEntities -from .oasis_profile_generator import OasisProfileGenerator, OasisAgentProfile -from .simulation_manager import SimulationManager, SimulationState, SimulationStatus -from .simulation_config_generator import ( - SimulationConfigGenerator, - SimulationParameters, - AgentActivityConfig, - TimeSimulationConfig, - EventConfig, - PlatformConfig -) -from .simulation_runner import ( - SimulationRunner, - SimulationRunState, - RunnerStatus, - AgentAction, - RoundSummary -) -from .zep_graph_memory_updater import ( - ZepGraphMemoryUpdater, - ZepGraphMemoryManager, - AgentActivity -) -from .simulation_ipc import ( - SimulationIPCClient, - SimulationIPCServer, - IPCCommand, - IPCResponse, - CommandType, - CommandStatus -) +from importlib import import_module -__all__ = [ - 'OntologyGenerator', - 'GraphBuilderService', - 'TextProcessor', - 'ZepEntityReader', - 'EntityNode', - 'FilteredEntities', - 'OasisProfileGenerator', - 'OasisAgentProfile', - 'SimulationManager', - 'SimulationState', - 'SimulationStatus', - 'SimulationConfigGenerator', - 'SimulationParameters', - 'AgentActivityConfig', - 'TimeSimulationConfig', - 'EventConfig', - 'PlatformConfig', - 'SimulationRunner', - 'SimulationRunState', - 'RunnerStatus', - 'AgentAction', - 'RoundSummary', - 'ZepGraphMemoryUpdater', - 'ZepGraphMemoryManager', - 'AgentActivity', - 'SimulationIPCClient', - 'SimulationIPCServer', - 'IPCCommand', - 'IPCResponse', - 'CommandType', - 'CommandStatus', -] +_EXPORTS = { + "OntologyGenerator": (".ontology_generator", "OntologyGenerator"), + "GraphBuilderService": (".graph_builder", "GraphBuilderService"), + "TextProcessor": (".text_processor", "TextProcessor"), + "ZepEntityReader": (".zep_entity_reader", "ZepEntityReader"), + "EntityNode": (".zep_entity_reader", "EntityNode"), + "FilteredEntities": (".zep_entity_reader", "FilteredEntities"), + "OasisProfileGenerator": (".oasis_profile_generator", "OasisProfileGenerator"), + "OasisAgentProfile": (".oasis_profile_generator", "OasisAgentProfile"), + "SimulationManager": (".simulation_manager", "SimulationManager"), + "SimulationState": (".simulation_manager", "SimulationState"), + "SimulationStatus": (".simulation_manager", "SimulationStatus"), + "SimulationConfigGenerator": (".simulation_config_generator", "SimulationConfigGenerator"), + "SimulationParameters": (".simulation_config_generator", "SimulationParameters"), + "AgentActivityConfig": (".simulation_config_generator", "AgentActivityConfig"), + "TimeSimulationConfig": (".simulation_config_generator", "TimeSimulationConfig"), + "EventConfig": (".simulation_config_generator", "EventConfig"), + "PlatformConfig": (".simulation_config_generator", "PlatformConfig"), + "SimulationRunner": (".simulation_runner", "SimulationRunner"), + "SimulationRunState": (".simulation_runner", "SimulationRunState"), + "RunnerStatus": (".simulation_runner", "RunnerStatus"), + "AgentAction": (".simulation_runner", "AgentAction"), + "RoundSummary": (".simulation_runner", "RoundSummary"), + "ZepGraphMemoryUpdater": (".zep_graph_memory_updater", "ZepGraphMemoryUpdater"), + "ZepGraphMemoryManager": (".zep_graph_memory_updater", "ZepGraphMemoryManager"), + "AgentActivity": (".memory_activity", "AgentActivity"), + "SimulationIPCClient": (".simulation_ipc", "SimulationIPCClient"), + "SimulationIPCServer": (".simulation_ipc", "SimulationIPCServer"), + "IPCCommand": (".simulation_ipc", "IPCCommand"), + "IPCResponse": (".simulation_ipc", "IPCResponse"), + "CommandType": (".simulation_ipc", "CommandType"), + "CommandStatus": (".simulation_ipc", "CommandStatus"), +} + + +def __getattr__(name: str): + try: + module_name, attribute_name = _EXPORTS[name] + except KeyError as exc: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from exc + module = import_module(module_name, __name__) + value = getattr(module, attribute_name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(_EXPORTS)) + + +__all__ = sorted(_EXPORTS) diff --git a/backend/app/services/artifact_store.py b/backend/app/services/artifact_store.py new file mode 100644 index 0000000..3728438 --- /dev/null +++ b/backend/app/services/artifact_store.py @@ -0,0 +1,87 @@ +"""Tenant-scoped artifact store abstraction. + +Wraps filesystem artifact persistence behind a small interface so project files, +simulation artifacts, and reports can later be moved to object storage without +changing callers. All paths are resolved under a tenant directory and reject +traversal/absolute components (fail-closed). +""" + +from __future__ import annotations + +import os +import re +from typing import Optional + +_SAFE_SEGMENT = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") + + +class ArtifactStore: + """Resolve tenant-scoped storage paths rooted under ``root``. + + ``path_for`` maps an owner/org and nested segments to an absolute path under + ``root//``. Path components are validated so a + caller can never escape the configured root. + """ + + def __init__(self, root: str): + if not isinstance(root, str) or not root.strip(): + raise ValueError("artifact_root_required") + self.root = os.path.realpath(root) + + def _validate_segments(self, segments): + cleaned = [] + for segment in segments: + if not isinstance(segment, str) or not _SAFE_SEGMENT.fullmatch(segment): + raise ValueError("invalid_artifact_path") + cleaned.append(segment) + return cleaned + + def path_for(self, *segments) -> str: + """Return a safe absolute path under the root for the given segments. + + The first segment is treated as the tenant/owner scope; every segment + must be a safe identifier (no separators, no dots-only, no traversal). + """ + if not segments: + raise ValueError("artifact_path_required") + cleaned = self._validate_segments(segments) + candidate = os.path.realpath(os.path.join(self.root, *cleaned)) + if os.path.commonpath([self.root, candidate]) != self.root: + raise ValueError("invalid_artifact_path") + return candidate + + def ensure_parent(self, path: str) -> None: + parent = os.path.dirname(path) + if parent: + os.makedirs(parent, exist_ok=True) + + def store_bytes(self, path: str, data: bytes) -> None: + self.ensure_parent(path) + with open(path, "wb") as handle: + handle.write(data) + + def read_bytes(self, path: str) -> Optional[bytes]: + if not self.exists(path): + return None + with open(path, "rb") as handle: + return handle.read() + + def exists(self, path: str) -> bool: + return os.path.isfile(path) + + def delete(self, path: str) -> bool: + if self.exists(path): + os.remove(path) + return True + return False + + +def default_artifact_store() -> ArtifactStore: + """Artifact store rooted at the configured upload root (filesystem backend). + + Later this factory can return an object-storage backed store without + changing callers. + """ + from ..config import Config + + return ArtifactStore(Config.UPLOAD_FOLDER) diff --git a/backend/app/services/audit_service.py b/backend/app/services/audit_service.py new file mode 100644 index 0000000..6c9b9e0 --- /dev/null +++ b/backend/app/services/audit_service.py @@ -0,0 +1,63 @@ +"""Durable, redacted audit event recording. + +Audit details never store secrets, tokens, password hashes, API keys, or raw +prompts — such keys are stripped before persisting. Events are tenant-scoped. +""" + +from __future__ import annotations + +from typing import Optional + +from sqlalchemy.orm import Session + +from ..models.operations import AuditLog + +_SENSITIVE_KEYS = {"password", "token", "api_key", "secret", "authorization", "prompt"} + + +class AuditService: + def __init__(self, session: Session): + self.session = session + + @staticmethod + def _redact(details) -> Optional[dict]: + if not isinstance(details, dict): + return None + return { + key: value + for key, value in details.items() + if key.lower() not in _SENSITIVE_KEYS + } + + def record( + self, + *, + organization_id: str, + actor_user_id: Optional[str] = None, + action: str, + target_type: str, + target_id: Optional[str] = None, + details: Optional[dict] = None, + ) -> str: + if not isinstance(organization_id, str) or not organization_id: + raise ValueError("organization_id_required") + entry = AuditLog( + organization_id=organization_id, + actor_user_id=actor_user_id, + action=action, + target_type=target_type, + target_id=target_id, + details=self._redact(details), + ) + self.session.add(entry) + self.session.flush() + return entry.id + + def list_for_organization(self, *, organization_id: str, limit: int = 100) -> list[AuditLog]: + return ( + self.session.query(AuditLog) + .filter(AuditLog.organization_id == organization_id) + .order_by(AuditLog.created_at.desc()) + .limit(min(max(int(limit), 1), 1000)) + .all() + ) diff --git a/backend/app/services/graph_builder.py b/backend/app/services/graph_builder.py index e40978a..f32a5ec 100644 --- a/backend/app/services/graph_builder.py +++ b/backend/app/services/graph_builder.py @@ -10,15 +10,15 @@ import threading from typing import Dict, Any, List, Optional, Callable from dataclasses import dataclass -from zep_cloud.client import Zep -from zep_cloud import EpisodeData, EntityEdgeSourceTarget - from ..config import Config from ..models.task import TaskManager, TaskStatus -from ..utils.zep_paging import fetch_all_nodes, fetch_all_edges +from ..utils.logger import get_logger from .text_processor import TextProcessor from ..utils.locale import t, get_locale, set_locale +logger = get_logger('crowdsight.graph_builder') +Zep = None + @dataclass class GraphInfo: @@ -43,13 +43,28 @@ class GraphBuilderService: 负责调用Zep API构建知识图谱 """ - def __init__(self, api_key: Optional[str] = None): + def __init__( + self, + api_key: Optional[str] = None, + *, + organization_id: Optional[str] = None, + owner_user_id: Optional[str] = None, + session_factory=None, + ): self.api_key = api_key or Config.ZEP_API_KEY if not self.api_key: raise ValueError("ZEP_API_KEY 未配置") - + + global Zep + if Zep is None: + from zep_cloud.client import Zep as ZepClient + + Zep = ZepClient + self.client = Zep(api_key=self.api_key) - self.task_manager = TaskManager() + self.organization_id = organization_id + self.owner_user_id = owner_user_id + self.task_manager = TaskManager(session_factory=session_factory) def build_graph_async( self, @@ -78,6 +93,8 @@ class GraphBuilderService: task_id = self.task_manager.create_task( task_type="graph_build", metadata={ + "organization_id": self.organization_id, + "owner_user_id": self.owner_user_id, "graph_name": graph_name, "chunk_size": chunk_size, "text_length": len(text), @@ -186,9 +203,12 @@ class GraphBuilderService: }) except Exception as e: - import traceback - error_msg = f"{str(e)}\n{traceback.format_exc()}" - self.task_manager.fail_task(task_id, error_msg) + logger.error( + "Graph build failed: task_id=%s error_type=%s", + task_id, + type(e).__name__, + ) + self.task_manager.fail_task(task_id, t('api.internalError')) def create_graph(self, name: str) -> str: """创建Zep图谱（公开方法）""" @@ -207,6 +227,7 @@ class GraphBuilderService: import warnings from typing import Optional from pydantic import Field + from zep_cloud import EntityEdgeSourceTarget from zep_cloud.external_clients.ontology import EntityModel, EntityText, EdgeModel # 抑制 Pydantic v2 关于 Field(default=None) 的警告 @@ -315,6 +336,8 @@ class GraphBuilderService: ) # 构建episode数据 + from zep_cloud import EpisodeData + episodes = [ EpisodeData(data=chunk, type="text") for chunk in batch_chunks @@ -402,6 +425,8 @@ class GraphBuilderService: def _get_graph_info(self, graph_id: str) -> GraphInfo: """获取图谱信息""" + from ..utils.zep_paging import fetch_all_edges, fetch_all_nodes + # 获取节点（分页） nodes = fetch_all_nodes(self.client, graph_id) @@ -433,6 +458,8 @@ class GraphBuilderService: Returns: 包含nodes和edges的字典，包括时间信息、属性等详细数据 """ + from ..utils.zep_paging import fetch_all_edges, fetch_all_nodes + nodes = fetch_all_nodes(self.client, graph_id) edges = fetch_all_edges(self.client, graph_id) diff --git a/backend/app/services/idempotency.py b/backend/app/services/idempotency.py new file mode 100644 index 0000000..9251f30 --- /dev/null +++ b/backend/app/services/idempotency.py @@ -0,0 +1,244 @@ +"""Database-backed idempotency for cookie-authenticated mutations.""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from functools import wraps +from typing import Any + +from flask import current_app, g, jsonify, make_response, request +from sqlalchemy import delete, select +from sqlalchemy.orm import Session + +from ..models.operations import IdempotencyRecord +from ..utils.api_errors import ApiError +from ..utils.locale import t + +_KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") + + +class IdempotencyConflict(ValueError): + """The key was reused for a different request body.""" + + +@dataclass(frozen=True) +class Reservation: + record: IdempotencyRecord + is_new: bool + + +class IdempotencyService: + DEFAULT_TTL_SECONDS = 24 * 60 * 60 + + def __init__(self, session: Session, *, ttl_seconds: int = DEFAULT_TTL_SECONDS): + if ttl_seconds < 60 or ttl_seconds > 7 * 24 * 60 * 60: + raise ValueError("invalid_idempotency_ttl") + self.session = session + self.ttl_seconds = ttl_seconds + + @staticmethod + def _request_hash(request_body: Any) -> str: + try: + encoded = json.dumps( + request_body, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise ValueError("invalid_idempotency_body") from exc + return hashlib.sha256(encoded).hexdigest() + + @staticmethod + def validate_key(key: str) -> str: + if not isinstance(key, str) or not _KEY_RE.fullmatch(key): + raise ValueError("invalid_idempotency_key") + return key + + def reserve( + self, + *, + organization_id: str, + user_id: str, + key: str, + request_body: Any, + ) -> Reservation: + if not organization_id or not user_id: + raise ValueError("invalid_idempotency_scope") + normalized_key = self.validate_key(key) + request_hash = self._request_hash(request_body) + now = datetime.now(timezone.utc) + existing = self.session.scalar( + select(IdempotencyRecord) + .where( + IdempotencyRecord.organization_id == organization_id, + IdempotencyRecord.user_id == user_id, + IdempotencyRecord.key == normalized_key, + ) + .with_for_update() + ) + if existing is not None: + expires_at = existing.expires_at + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=timezone.utc) + else: + expires_at = None + if existing is not None and expires_at is not None and expires_at <= now: + self.session.execute(delete(IdempotencyRecord).where(IdempotencyRecord.id == existing.id)) + self.session.flush() + existing = None + if existing is not None: + if existing.request_hash != request_hash: + raise IdempotencyConflict("idempotency_key_reused") + return Reservation(record=existing, is_new=False) + + record = IdempotencyRecord( + organization_id=organization_id, + user_id=user_id, + key=normalized_key, + request_hash=request_hash, + status="reserved", + expires_at=now + timedelta(seconds=self.ttl_seconds), + ) + self.session.add(record) + self.session.flush() + return Reservation(record=record, is_new=True) + + def complete(self, record: IdempotencyRecord, *, status_code: int, body: dict | list) -> None: + if status_code < 100 or status_code > 599: + raise ValueError("invalid_response_status") + record.status = "completed" + record.response_status = status_code + record.response_body = body + record.completed_at = datetime.now(timezone.utc) + self.session.flush() + + def fail(self, record: IdempotencyRecord, *, status_code: int, body: dict | list) -> None: + if status_code < 400 or status_code > 599: + raise ValueError("invalid_failure_status") + record.status = "failed" + record.response_status = status_code + record.response_body = body + record.completed_at = datetime.now(timezone.utc) + self.session.flush() + + +def _stream_sha256(stream) -> str: + try: + start_position = stream.tell() + except (AttributeError, OSError, ValueError) as exc: + raise ValueError("invalid_idempotency_body") from exc + + digest = hashlib.sha256() + try: + while True: + chunk = stream.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + except (AttributeError, OSError, TypeError, ValueError) as exc: + raise ValueError("invalid_idempotency_body") from exc + finally: + try: + stream.seek(start_position) + except (AttributeError, OSError, ValueError) as exc: + raise ValueError("invalid_idempotency_body") from exc + return digest.hexdigest() + + +def _request_fingerprint_payload() -> Any: + if request.is_json: + body = request.get_json(silent=True) + elif request.form or request.files: + files = [ + { + "field": field, + "filename": file.filename or "", + "content_type": file.content_type or "", + "size": request.content_length or 0, + "content_sha256": _stream_sha256(file.stream), + } + for field, files in request.files.lists() + for file in files + ] + body = {"form": request.form.to_dict(flat=False), "files": files} + else: + body = {} + return { + "method": request.method, + "path": request.path, + "query": request.args.to_dict(flat=False), + "body": body, + } + + +def idempotent(view): + """Require and persist an ``Idempotency-Key`` for a JSON mutation.""" + + @wraps(view) + def wrapped(*args, **kwargs): + key = request.headers.get("Idempotency-Key", "") + if not key: + raise ApiError("idempotency_required", 400, "api.idempotencyRequired") + session = getattr(g, "db_session", None) + managed_session = False + if session is None: + factory = current_app.extensions.get("crowdsight_session_factory") + if not callable(factory): + raise ApiError("auth_unavailable", 503, "api.internalError") + session = factory() + managed_session = True + context = getattr(g, "auth_context", None) + if context is None: + if managed_session: + session.close() + raise ApiError("auth_unavailable", 503, "api.internalError") + try: + try: + reservation = IdempotencyService(session).reserve( + organization_id=context.organization.id, + user_id=context.user.id, + key=key, + request_body=_request_fingerprint_payload(), + ) + except IdempotencyConflict as exc: + raise ApiError("idempotency_key_reused", 409, "api.idempotencyConflict") from exc + except ValueError as exc: + code = str(exc) + if code == "invalid_idempotency_key": + raise ApiError(code, 400, "api.idempotencyInvalid") from exc + raise ApiError("invalid_request", 400, "api.requestError") from exc + + if not reservation.is_new: + record = reservation.record + if record.status == "completed" and record.response_body is not None: + return jsonify(record.response_body), record.response_status or 200 + if record.status == "failed" and record.response_body is not None: + return jsonify(record.response_body), record.response_status or 500 + raise ApiError("idempotency_in_progress", 409, "api.idempotencyInProgress") + + response = make_response(view(*args, **kwargs)) + body = response.get_json(silent=True) + if not isinstance(body, (dict, list)): + raise ApiError("idempotency_response_invalid", 500, "api.internalError") + IdempotencyService(session).complete( + reservation.record, + status_code=response.status_code, + body=body, + ) + if managed_session: + session.commit() + return response + except Exception: + if managed_session: + session.rollback() + raise + finally: + if managed_session: + session.close() + + return wrapped diff --git a/backend/app/services/identity.py b/backend/app/services/identity.py new file mode 100644 index 0000000..f894ef5 --- /dev/null +++ b/backend/app/services/identity.py @@ -0,0 +1,260 @@ +"""Tenant-scoped identity repository and password service.""" + +from __future__ import annotations + +import hashlib +import secrets +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError +from sqlalchemy import select +from sqlalchemy.orm import Session + +from ..models.saas import AuthSession, Membership, Organization, User +from ..security.policy import Role + + +class PasswordService: + """Argon2id password hashing with no plaintext fallback.""" + + _hasher = PasswordHasher() + + @classmethod + def hash_password(cls, password: str) -> str: + if not isinstance(password, str) or len(password) < 12: + raise ValueError("password_too_short") + return cls._hasher.hash(password) + + @classmethod + def verify_password(cls, password_hash: str, password: str) -> bool: + if not isinstance(password_hash, str) or not password_hash: + return False + try: + return cls._hasher.verify(password_hash, password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False + + +class IdentityRepository: + """Flush-only repository; the caller owns transaction boundaries.""" + + def __init__(self, session: Session): + self.session = session + + @staticmethod + def normalize_email(email: str) -> str: + if not isinstance(email, str): + raise ValueError("invalid_email") + normalized = email.strip().casefold() + if "@" not in normalized or normalized.startswith("@") or normalized.endswith("@"): + raise ValueError("invalid_email") + return normalized + + @staticmethod + def normalize_slug(slug: str) -> str: + if not isinstance(slug, str): + raise ValueError("invalid_slug") + normalized = slug.strip().casefold() + if not normalized or any(char not in "abcdefghijklmnopqrstuvwxyz0123456789-" for char in normalized): + raise ValueError("invalid_slug") + return normalized + + def create_organization(self, *, name: str, slug: str) -> Organization: + if not isinstance(name, str) or not name.strip(): + raise ValueError("invalid_organization_name") + organization = Organization(name=name.strip(), slug=self.normalize_slug(slug)) + self.session.add(organization) + self.session.flush() + return organization + + def create_user(self, *, email: str, password_hash: str | None = None) -> User: + user = User( + email_normalized=self.normalize_email(email), + password_hash=password_hash or "!invite_pending", + ) + self.session.add(user) + self.session.flush() + return user + + def create_membership(self, user_id: str, organization_id: str, role: Role | str) -> Membership: + try: + normalized_role = role if isinstance(role, Role) else Role(role) + except (TypeError, ValueError) as exc: + raise ValueError("invalid_role") from exc + + membership = Membership( + user_id=user_id, + organization_id=organization_id, + role=normalized_role, + ) + self.session.add(membership) + self.session.flush() + return membership + + def get_organization(self, organization_id: str) -> Organization | None: + return self.session.scalar( + select(Organization).where(Organization.id == organization_id) + ) + + def get_organization_by_slug(self, slug: str) -> Organization | None: + normalized = self.normalize_slug(slug) + return self.session.scalar( + select(Organization).where(Organization.slug == normalized) + ) + + def list_active_memberships(self, user_id: str) -> list[tuple[Membership, Organization]]: + return list( + self.session.execute( + select(Membership, Organization) + .join(Organization, Organization.id == Membership.organization_id) + .where( + Membership.user_id == user_id, + Membership.status == "active", + Organization.status == "active", + ) + .order_by(Organization.slug) + ).all() + ) + + def get_user_by_email(self, email: str) -> User | None: + normalized = self.normalize_email(email) + return self.session.scalar( + select(User).where(User.email_normalized == normalized) + ) + + def get_user_for_org(self, user_id: str, organization_id: str) -> User | None: + """Tenant-scoped read; wrong-org IDs return None without disclosure.""" + return self.session.scalar( + select(User) + .join(Membership, Membership.user_id == User.id) + .where( + User.id == user_id, + Membership.organization_id == organization_id, + Membership.status == "active", + User.status == "active", + ) + ) + + def list_users_with_memberships(self, organization_id: str) -> list[tuple[User, Membership]]: + return list( + self.session.execute( + select(User, Membership) + .join(Membership, Membership.user_id == User.id) + .where( + Membership.organization_id == organization_id, + Membership.status == "active", + User.status == "active", + ) + .order_by(User.email_normalized) + ).all() + ) + + def list_users(self, organization_id: str) -> list[User]: + """Return only active users with active membership in the tenant.""" + return list( + self.session.scalars( + select(User) + .join(Membership, Membership.user_id == User.id) + .where( + Membership.organization_id == organization_id, + Membership.status == "active", + User.status == "active", + ) + .order_by(User.email_normalized) + ) + ) + + +@dataclass(frozen=True) +class SessionContext: + user: User + membership: Membership + organization: Organization + + +class SessionService: + """Opaque, revocable session token operations; caller owns commits.""" + + DEFAULT_TTL_SECONDS = 60 * 60 * 12 + + @staticmethod + def _hash_token(raw_token: str) -> str: + return hashlib.sha256(raw_token.encode("utf-8")).hexdigest() + + @classmethod + def create( + cls, + session: Session, + user: User, + membership_id: str, + ttl_seconds: int | None = None, + ) -> tuple[str, AuthSession]: + if not isinstance(membership_id, str) or not membership_id: + raise ValueError("invalid_membership") + ttl = ttl_seconds or cls.DEFAULT_TTL_SECONDS + if ttl < 60: + raise ValueError("invalid_session_ttl") + + membership = session.scalar( + select(Membership).where( + Membership.id == membership_id, + Membership.user_id == user.id, + Membership.status == "active", + ) + ) + if membership is None: + raise ValueError("invalid_membership") + + raw_token = secrets.token_urlsafe(32) + stored = AuthSession( + user_id=user.id, + membership_id=membership_id, + token_hash=cls._hash_token(raw_token), + auth_version=user.auth_version, + expires_at=datetime.now(timezone.utc) + timedelta(seconds=ttl), + ) + session.add(stored) + session.flush() + return raw_token, stored + + @classmethod + def resolve(cls, session: Session, raw_token: str) -> SessionContext | None: + if not isinstance(raw_token, str) or not raw_token: + return None + + now = datetime.now(timezone.utc) + row = session.execute( + select(AuthSession, User, Membership, Organization) + .join(User, User.id == AuthSession.user_id) + .join(Membership, Membership.id == AuthSession.membership_id) + .join(Organization, Organization.id == Membership.organization_id) + .where( + AuthSession.token_hash == cls._hash_token(raw_token), + AuthSession.revoked_at.is_(None), + AuthSession.expires_at > now, + AuthSession.auth_version == User.auth_version, + User.status == "active", + Membership.status == "active", + Organization.status == "active", + ) + ).first() + if row is None: + return None + + stored, user, membership, organization = row + stored.last_seen_at = now + return SessionContext(user=user, membership=membership, organization=organization) + + @classmethod + def revoke(cls, session: Session, raw_token: str) -> bool: + if not isinstance(raw_token, str) or not raw_token: + return False + stored = session.scalar( + select(AuthSession).where(AuthSession.token_hash == cls._hash_token(raw_token)) + ) + if stored is None or stored.revoked_at is not None: + return False + stored.revoked_at = datetime.now(timezone.utc) + session.flush() + return True diff --git a/backend/app/services/job_queue.py b/backend/app/services/job_queue.py new file mode 100644 index 0000000..8202e89 --- /dev/null +++ b/backend/app/services/job_queue.py @@ -0,0 +1,82 @@ +"""Durable job queue: claim/complete/fail lifecycle over the ``jobs`` table. + +This is the portable core a worker topology builds on; it does not require a +broker. For PostgreSQL production this should issue ``SELECT ... FOR UPDATE`` +plus ``UPDATE ... WHERE status='queued'`` to claim atomically; the SQLite +fallback below uses a synchronized in-process write for local/tests. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Callable, Optional + +from sqlalchemy.orm import Session + +from ..models.operations import Job, JobStatus + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class JobQueue: + """Flush-only queue: caller owns the transaction boundary.""" + + def __init__(self, session: Session): + self.session = session + self._handlers: dict[str, Callable] = {} + + def register_handler(self, operation: str, handler) -> None: + """Register a callable handler for an operation (worker plugin point).""" + self._handlers[operation] = handler + + def dispatch(self, job: Job, *, payload=None): + """Invoke the registered handler for ``job.operation``. + + Returns the handler result. Raises ``ValueError`` when no handler is + registered so the worker can fail the job. + """ + handler = self._handlers.get(job.operation) + if handler is None: + raise ValueError(f"no_handler_for_operation: {job.operation}") + return handler(payload, job) + + def claim_next_job( + self, *, worker_id: str = "worker", organization_id: Optional[str] = None + ) -> Optional[Job]: + """Claim the next queued job for a worker. + + For the given optional organization scope, atomically flip one queued + job to ``running`` and return it; returns None when nothing is claimable. + """ + query = self.session.query(Job).filter(Job.status == JobStatus.QUEUED.value) + if organization_id is not None: + query = query.filter(Job.organization_id == organization_id) + job = query.order_by(Job.created_at.asc()).first() + if job is None: + return None + job.status = JobStatus.RUNNING.value + job.message = f"claimed by {worker_id}" + self.session.flush() + return job + + def complete_job(self, job_id: str, result=None, message: str = "") -> None: + job = self.session.get(Job, job_id) + if job is None: + raise ValueError("job_not_found") + job.status = JobStatus.SUCCEEDED.value + job.result = result + job.message = message or job.message + job.finished_at = _utc_now() + self.session.flush() + + def fail_job(self, job_id: str, error_code: str, message: str = "") -> None: + job = self.session.get(Job, job_id) + if job is None: + raise ValueError("job_not_found") + job.status = JobStatus.FAILED.value + job.error_code = error_code + job.message = message or job.message + job.finished_at = _utc_now() + self.session.flush() diff --git a/backend/app/services/local_graph_builder.py b/backend/app/services/local_graph_builder.py new file mode 100644 index 0000000..9f2121a --- /dev/null +++ b/backend/app/services/local_graph_builder.py @@ -0,0 +1,197 @@ +"""Tenant-scoped graph builder backed by the local memory repository.""" + +from __future__ import annotations + +from typing import Any, Callable, Iterable +from uuid import uuid4 + +from .memory_repository import SqlAlchemyMemoryRepository +from .memory_service import MemoryExtractionService +from ..utils.locale import get_locale +from ..utils.language_policy import normalize_locale +from ..utils.logger import get_logger + +logger = get_logger("crowdsight.local_graph_builder") + + +ProgressCallback = Callable[[str, float], None] + + +class LocalGraphBuilderService: + """Build a graph synchronously into local durable memory. + + The public methods intentionally mirror ``GraphBuilderService`` so the API + can switch storage backends without giving either backend authorization + authority. Every repository instance is created with the request actor's + organization and the graph being processed. + """ + + def __init__( + self, + session_factory, + *, + organization_id: str, + project_id: str, + extraction_client=None, + extraction_service: MemoryExtractionService | None = None, + language: str | None = None, + ): + if not callable(session_factory): + raise ValueError("memory_session_factory_required") + if not isinstance(organization_id, str) or not organization_id.strip(): + raise ValueError("memory_builder_organization_required") + if not isinstance(project_id, str) or not project_id.strip(): + raise ValueError("memory_builder_project_required") + self.session_factory = session_factory + self.organization_id = organization_id + self.project_id = project_id + self.language = normalize_locale(language or get_locale()) + self.extraction_service = ( + extraction_service + if extraction_service is not None + else MemoryExtractionService(extraction_client) if extraction_client is not None else None + ) + self._ontology_by_graph: dict[str, dict[str, Any]] = {} + + def _repository(self, session, graph_id: str) -> SqlAlchemyMemoryRepository: + return SqlAlchemyMemoryRepository( + session, + organization_id=self.organization_id, + graph_id=graph_id, + ) + + @staticmethod + def _validate_graph_id(graph_id: str) -> str: + if not isinstance(graph_id, str) or not graph_id.strip(): + raise ValueError("invalid_graph_id") + return graph_id.strip() + + def create_graph(self, name: str = "") -> str: + """Create a new graph and return its durable ID.""" + graph_id = f"crowdsight_{uuid4().hex[:24]}" + with self.session_factory() as session: + repository = self._repository(session, graph_id) + repository.create_graph(project_id=self.project_id) + session.commit() + logger.info("Created local memory graph %s", graph_id) + return graph_id + + def set_ontology(self, graph_id: str, ontology: dict[str, Any]) -> None: + graph_id = self._validate_graph_id(graph_id) + if not isinstance(ontology, dict): + raise ValueError("invalid_ontology") + with self.session_factory() as session: + repository = self._repository(session, graph_id) + repository.update_graph_ontology(ontology) + session.commit() + self._ontology_by_graph[graph_id] = dict(ontology) + + def add_text_batches( + self, + graph_id: str, + text_batches: Iterable[str], + batch_size: int = 3, + progress_callback: ProgressCallback | None = None, + ) -> list[str]: + graph_id = self._validate_graph_id(graph_id) + extraction_service = self.extraction_service + if extraction_service is None: + raise ValueError("memory_extraction_client_required") + if isinstance(batch_size, bool) or not isinstance(batch_size, int) or batch_size < 1: + raise ValueError("invalid_batch_size") + chunks = list(text_batches) + if any(not isinstance(chunk, str) or not chunk.strip() for chunk in chunks): + raise ValueError("invalid_text_batch") + + episode_ids: list[str] = [] + total = len(chunks) + for index, chunk in enumerate(chunks): + with self.session_factory() as session: + repository = self._repository(session, graph_id) + graph = repository.get_graph() + ontology = dict(self._ontology_by_graph.get(graph_id) or graph.ontology) + result = extraction_service.extract( + language=self.language, + ontology=ontology, + episode_text=chunk, + ) + ingest_result = extraction_service.persist( + repository, + result, + source_type="text", + source_ref=f"episode_{index}", + episode_text=chunk, + ) + episode = repository.get_episode(source_type="text", source_ref=f"episode_{index}") + session.commit() + if episode is None: + raise RuntimeError("local_episode_persist_failed") + episode_ids.append(episode.id) + if progress_callback: + progress_callback( + f"Processed local memory chunk {index + 1}/{total}", + (index + 1) / total if total else 1.0, + ) + logger.debug( + "Processed local graph chunk %s: entities=%s edges=%s", + index, + ingest_result.entity_count, + ingest_result.edge_count, + ) + return episode_ids + + def _wait_for_episodes( + self, + episode_ids: list[str], + progress_callback: ProgressCallback | None = None, + ) -> None: + """Local extraction is committed synchronously; verify IDs instead of polling Zep.""" + if not isinstance(episode_ids, list): + raise ValueError("invalid_episode_ids") + if progress_callback: + progress_callback("Local memory processing complete", 1.0) + + def get_graph_data(self, graph_id: str) -> dict[str, Any]: + graph_id = self._validate_graph_id(graph_id) + with self.session_factory() as session: + repository = self._repository(session, graph_id) + graph = repository.get_graph() + nodes = repository.list_nodes(limit=10_000) + edges = repository.list_edges(limit=20_000) + return { + "graph_id": graph.id, + "node_count": len(nodes), + "edge_count": len(edges), + "nodes": [ + { + "uuid": node.id, + "name": node.canonical_name, + "labels": list(node.labels or []), + "summary": node.summary or "", + "attributes": dict(node.attributes or {}), + } + for node in nodes + ], + "edges": [ + { + "uuid": edge.id, + "name": edge.relation, + "fact": edge.fact, + "source_node_uuid": edge.source_node_id, + "target_node_uuid": edge.target_node_id, + "attributes": dict(edge.attributes or {}), + } + for edge in edges + ], + } + + def delete_graph(self, graph_id: str) -> None: + """Delete a graph only when it belongs to this organization scope.""" + graph_id = self._validate_graph_id(graph_id) + with self.session_factory() as session: + repository = self._repository(session, graph_id) + graph = repository.get_graph() + session.delete(graph) + session.commit() + self._ontology_by_graph.pop(graph_id, None) + logger.info("Deleted local memory graph %s", graph_id) diff --git a/backend/app/services/local_graph_memory_updater.py b/backend/app/services/local_graph_memory_updater.py new file mode 100644 index 0000000..c1692c0 --- /dev/null +++ b/backend/app/services/local_graph_memory_updater.py @@ -0,0 +1,177 @@ +"""Tenant-scoped local persistence for simulation activities. + +This module intentionally does not import or construct a Zep client. It keeps +runtime activity updates behind the same small manager interface used by the +legacy runner, while storing each activity as a durable local memory episode. +""" + +from __future__ import annotations + +import threading +from datetime import datetime +from typing import Any, Dict, Optional, cast + +from sqlalchemy.orm import Session + +from .memory_repository import SqlAlchemyMemoryRepository +from ..utils.logger import get_logger + +logger = get_logger("crowdsight.local_graph_memory_updater") + + +class LocalGraphMemoryUpdater: + """Persist simulation activities in a tenant-scoped local graph.""" + + def __init__( + self, + simulation_id: str, + graph_id: str, + organization_id: str, + session_factory, + ): + for name, value in ( + ("simulation_id", simulation_id), + ("graph_id", graph_id), + ("organization_id", organization_id), + ): + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"local_memory_{name}_required") + if not callable(session_factory): + raise ValueError("memory_session_factory_required") + + self.simulation_id = simulation_id + self.graph_id = graph_id + self.organization_id = organization_id + self.session_factory = session_factory + self._running = False + self._sequence = 0 + self._lock = threading.Lock() + self._total_activities = 0 + self._total_sent = 0 + self._total_items_sent = 0 + self._failed_count = 0 + self._skipped_count = 0 + + def start(self) -> None: + self._running = True + + def stop(self) -> None: + self._running = False + + def add_activity(self, activity: Any) -> None: + if not callable(getattr(activity, "to_episode_text", None)): + raise ValueError("invalid_agent_activity") + if activity.action_type == "DO_NOTHING": + self._skipped_count += 1 + return + + with self._lock: + self._sequence += 1 + sequence = self._sequence + self._total_activities += 1 + + try: + episode_text = activity.to_episode_text() + source_ref = ( + f"{self.simulation_id}:{activity.platform}:{activity.round_num}:" + f"{activity.agent_id}:{sequence}" + ) + session: Session = cast(Session, self.session_factory()) + try: + repository = SqlAlchemyMemoryRepository( + session, + organization_id=self.organization_id, + graph_id=self.graph_id, + ) + repository.add_episode( + source_type="simulation_activity", + source_ref=source_ref, + normalized_text=episode_text, + summary=episode_text, + extractor_version="runtime-v1", + ) + session.commit() + finally: + session.close() + self._total_sent += 1 + self._total_items_sent += 1 + except Exception: + self._failed_count += 1 + raise + + def add_activity_from_dict(self, data: Dict[str, Any], platform: str) -> None: + if "event_type" in data: + return + from .memory_activity import AgentActivity + + self.add_activity( + AgentActivity( + platform=platform, + agent_id=data.get("agent_id", 0), + agent_name=data.get("agent_name", ""), + action_type=data.get("action_type", ""), + action_args=data.get("action_args", {}), + round_num=data.get("round", 0), + timestamp=data.get("timestamp", datetime.now().isoformat()), + ) + ) + + def get_stats(self) -> Dict[str, Any]: + return { + "total_activities": self._total_activities, + "batches_sent": self._total_sent, + "items_sent": self._total_items_sent, + "failed_count": self._failed_count, + "skipped_count": self._skipped_count, + "queue_size": 0, + "buffer_sizes": {}, + "running": self._running, + } + + +class LocalGraphMemoryManager: + """Manage one local updater per simulation.""" + + _updaters: Dict[str, LocalGraphMemoryUpdater] = {} + _lock = threading.Lock() + + @classmethod + def create_updater( + cls, + simulation_id: str, + graph_id: str, + *, + organization_id: str, + session_factory, + ) -> LocalGraphMemoryUpdater: + with cls._lock: + existing = cls._updaters.get(simulation_id) + if existing is not None: + existing.stop() + updater = LocalGraphMemoryUpdater( + simulation_id=simulation_id, + graph_id=graph_id, + organization_id=organization_id, + session_factory=session_factory, + ) + updater.start() + cls._updaters[simulation_id] = updater + return updater + + @classmethod + def get_updater(cls, simulation_id: str) -> Optional[LocalGraphMemoryUpdater]: + return cls._updaters.get(simulation_id) + + @classmethod + def stop_updater(cls, simulation_id: str) -> None: + with cls._lock: + updater = cls._updaters.pop(simulation_id, None) + if updater is not None: + updater.stop() + + @classmethod + def stop_all(cls) -> None: + with cls._lock: + simulation_ids = list(cls._updaters) + for simulation_id in simulation_ids: + cls.stop_updater(simulation_id) diff --git a/backend/app/services/memory_activity.py b/backend/app/services/memory_activity.py new file mode 100644 index 0000000..0211704 --- /dev/null +++ b/backend/app/services/memory_activity.py @@ -0,0 +1,184 @@ +"""Shared simulation activity contract for local and Zep memory backends.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict + + +@dataclass +class AgentActivity: + """Agent活动记录""" + platform: str # twitter / reddit + agent_id: int + agent_name: str + action_type: str # CREATE_POST, LIKE_POST, etc. + action_args: Dict[str, Any] + round_num: int + timestamp: str + + def to_episode_text(self) -> str: + """ + 将活动转换为可以发送给Zep的文本描述 + + 采用自然语言描述格式，让Zep能够从中提取实体和关系 + 不添加模拟相关的前缀，避免误导图谱更新 + """ + # 根据不同的动作类型生成不同的描述 + action_descriptions = { + "CREATE_POST": self._describe_create_post, + "LIKE_POST": self._describe_like_post, + "DISLIKE_POST": self._describe_dislike_post, + "REPOST": self._describe_repost, + "QUOTE_POST": self._describe_quote_post, + "FOLLOW": self._describe_follow, + "CREATE_COMMENT": self._describe_create_comment, + "LIKE_COMMENT": self._describe_like_comment, + "DISLIKE_COMMENT": self._describe_dislike_comment, + "SEARCH_POSTS": self._describe_search, + "SEARCH_USER": self._describe_search_user, + "MUTE": self._describe_mute, + } + + describe_func = action_descriptions.get(self.action_type, self._describe_generic) + description = describe_func() + + # 直接返回 "agent名称: 活动描述" 格式，不添加模拟前缀 + return f"{self.agent_name}: {description}" + + def _describe_create_post(self) -> str: + content = self.action_args.get("content", "") + if content: + return f"发布了一条帖子：「{content}」" + return "发布了一条帖子" + + def _describe_like_post(self) -> str: + """点赞帖子 - 包含帖子原文和作者信息""" + post_content = self.action_args.get("post_content", "") + post_author = self.action_args.get("post_author_name", "") + + if post_content and post_author: + return f"点赞了{post_author}的帖子：「{post_content}」" + elif post_content: + return f"点赞了一条帖子：「{post_content}」" + elif post_author: + return f"点赞了{post_author}的一条帖子" + return "点赞了一条帖子" + + def _describe_dislike_post(self) -> str: + """踩帖子 - 包含帖子原文和作者信息""" + post_content = self.action_args.get("post_content", "") + post_author = self.action_args.get("post_author_name", "") + + if post_content and post_author: + return f"踩了{post_author}的帖子：「{post_content}」" + elif post_content: + return f"踩了一条帖子：「{post_content}」" + elif post_author: + return f"踩了{post_author}的一条帖子" + return "踩了一条帖子" + + def _describe_repost(self) -> str: + """转发帖子 - 包含原帖内容和作者信息""" + original_content = self.action_args.get("original_content", "") + original_author = self.action_args.get("original_author_name", "") + + if original_content and original_author: + return f"转发了{original_author}的帖子：「{original_content}」" + elif original_content: + return f"转发了一条帖子：「{original_content}」" + elif original_author: + return f"转发了{original_author}的一条帖子" + return "转发了一条帖子" + + def _describe_quote_post(self) -> str: + """引用帖子 - 包含原帖内容、作者信息和引用评论""" + original_content = self.action_args.get("original_content", "") + original_author = self.action_args.get("original_author_name", "") + quote_content = self.action_args.get("quote_content", "") or self.action_args.get("content", "") + + base = "" + if original_content and original_author: + base = f"引用了{original_author}的帖子「{original_content}」" + elif original_content: + base = f"引用了一条帖子「{original_content}」" + elif original_author: + base = f"引用了{original_author}的一条帖子" + else: + base = "引用了一条帖子" + + if quote_content: + base += f"，并评论道：「{quote_content}」" + return base + + def _describe_follow(self) -> str: + """关注用户 - 包含被关注用户的名称""" + target_user_name = self.action_args.get("target_user_name", "") + + if target_user_name: + return f"关注了用户「{target_user_name}」" + return "关注了一个用户" + + def _describe_create_comment(self) -> str: + """发表评论 - 包含评论内容和所评论的帖子信息""" + content = self.action_args.get("content", "") + post_content = self.action_args.get("post_content", "") + post_author = self.action_args.get("post_author_name", "") + + if content: + if post_content and post_author: + return f"在{post_author}的帖子「{post_content}」下评论道：「{content}」" + elif post_content: + return f"在帖子「{post_content}」下评论道：「{content}」" + elif post_author: + return f"在{post_author}的帖子下评论道：「{content}」" + return f"评论道：「{content}」" + return "发表了评论" + + def _describe_like_comment(self) -> str: + """点赞评论 - 包含评论内容和作者信息""" + comment_content = self.action_args.get("comment_content", "") + comment_author = self.action_args.get("comment_author_name", "") + + if comment_content and comment_author: + return f"点赞了{comment_author}的评论：「{comment_content}」" + elif comment_content: + return f"点赞了一条评论：「{comment_content}」" + elif comment_author: + return f"点赞了{comment_author}的一条评论" + return "点赞了一条评论" + + def _describe_dislike_comment(self) -> str: + """踩评论 - 包含评论内容和作者信息""" + comment_content = self.action_args.get("comment_content", "") + comment_author = self.action_args.get("comment_author_name", "") + + if comment_content and comment_author: + return f"踩了{comment_author}的评论：「{comment_content}」" + elif comment_content: + return f"踩了一条评论：「{comment_content}」" + elif comment_author: + return f"踩了{comment_author}的一条评论" + return "踩了一条评论" + + def _describe_search(self) -> str: + """搜索帖子 - 包含搜索关键词""" + query = self.action_args.get("query", "") or self.action_args.get("keyword", "") + return f"搜索了「{query}」" if query else "进行了搜索" + + def _describe_search_user(self) -> str: + """搜索用户 - 包含搜索关键词""" + query = self.action_args.get("query", "") or self.action_args.get("username", "") + return f"搜索了用户「{query}」" if query else "搜索了用户" + + def _describe_mute(self) -> str: + """屏蔽用户 - 包含被屏蔽用户的名称""" + target_user_name = self.action_args.get("target_user_name", "") + + if target_user_name: + return f"屏蔽了用户「{target_user_name}」" + return "屏蔽了一个用户" + + def _describe_generic(self) -> str: + # 对于未知的动作类型，生成通用描述 + return f"执行了{self.action_type}操作" diff --git a/backend/app/services/memory_entity_reader.py b/backend/app/services/memory_entity_reader.py new file mode 100644 index 0000000..0dbdec5 --- /dev/null +++ b/backend/app/services/memory_entity_reader.py @@ -0,0 +1,242 @@ +"""Local entity-reader compatibility adapter for the legacy Zep consumer shape.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Optional + +from .memory_repository import SqlAlchemyMemoryRepository + + +@dataclass +class LocalEntityNode: + uuid: str + name: str + labels: list[str] + summary: str + attributes: dict[str, Any] + related_edges: list[dict[str, Any]] = field(default_factory=list) + related_nodes: list[dict[str, Any]] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "uuid": self.uuid, + "name": self.name, + "labels": self.labels, + "summary": self.summary, + "attributes": self.attributes, + "related_edges": self.related_edges, + "related_nodes": self.related_nodes, + } + + def get_entity_type(self) -> Optional[str]: + return next((label for label in self.labels if label not in {"Entity", "Node"}), None) + + +@dataclass +class LocalFilteredEntities: + entities: list[LocalEntityNode] + entity_types: set[str] + total_count: int + filtered_count: int + + def to_dict(self) -> dict[str, Any]: + return { + "entities": [entity.to_dict() for entity in self.entities], + "entity_types": sorted(self.entity_types), + "total_count": self.total_count, + "filtered_count": self.filtered_count, + } + + +class LocalEntityReader: + def __init__( + self, + session_or_repository, + *, + organization_id: str | None = None, + graph_id: str | None = None, + owns_session: bool = False, + ): + self.owns_session = False + if isinstance(session_or_repository, SqlAlchemyMemoryRepository): + self.repository = session_or_repository + else: + if organization_id is None or graph_id is None: + raise ValueError("memory_reader_scope_required") + self.repository = SqlAlchemyMemoryRepository( + session_or_repository, + organization_id=organization_id, + graph_id=graph_id, + ) + self.owns_session = owns_session + + def _validate_graph_id(self, graph_id: str | None = None) -> None: + if graph_id is not None and graph_id != self.repository.graph_id: + raise ValueError("memory_graph_scope_conflict") + + def close(self) -> None: + if self.owns_session: + self.repository.session.close() + self.owns_session = False + + @staticmethod + def _node_dict(node) -> dict[str, Any]: + return { + "uuid": node.id, + "name": node.canonical_name, + "labels": list(node.labels or []), + "summary": node.summary or "", + "attributes": dict(node.attributes or {}), + } + + def _entity(self, node, *, enrich_with_edges: bool = True) -> LocalEntityNode: + related_edges: list[dict[str, Any]] = [] + related_nodes: list[dict[str, Any]] = [] + if enrich_with_edges: + all_nodes = {item.id: item for item in self.repository.list_nodes()} + for edge in self.repository.get_node_edges(node.id): + source = all_nodes.get(edge.source_node_id) + target = all_nodes.get(edge.target_node_id) + if edge.source_node_id == node.id: + related_edges.append( + { + "direction": "outgoing", + "edge_name": edge.relation, + "fact": edge.fact, + "target_node_uuid": edge.target_node_id, + } + ) + related_node = target + else: + related_edges.append( + { + "direction": "incoming", + "edge_name": edge.relation, + "fact": edge.fact, + "source_node_uuid": edge.source_node_id, + } + ) + related_node = source + if related_node is not None: + related_nodes.append( + { + "uuid": related_node.id, + "name": related_node.canonical_name, + "labels": list(related_node.labels or []), + "summary": related_node.summary or "", + } + ) + return LocalEntityNode( + uuid=node.id, + name=node.canonical_name, + labels=list(node.labels or []), + summary=node.summary or "", + attributes=dict(node.attributes or {}), + related_edges=related_edges, + related_nodes=related_nodes, + ) + + def get_all_nodes(self, graph_id: str | None = None) -> list[dict[str, Any]]: + self._validate_graph_id(graph_id) + return [self._node_dict(node) for node in self.repository.list_nodes()] + + def get_all_edges(self, graph_id: str | None = None) -> list[dict[str, Any]]: + self._validate_graph_id(graph_id) + return [ + { + "uuid": edge.id, + "name": edge.relation, + "fact": edge.fact, + "source_node_uuid": edge.source_node_id, + "target_node_uuid": edge.target_node_id, + "attributes": dict(edge.attributes or {}), + } + for edge in self.repository.list_edges() + ] + + def filter_defined_entities( + self, + graph_id: str | None = None, + *, + defined_entity_types: Optional[list[str]] = None, + enrich_with_edges: bool = True, + ) -> LocalFilteredEntities: + self._validate_graph_id(graph_id) + nodes = self.repository.list_nodes() + allowed = set(defined_entity_types or []) + entity_types: set[str] = set() + entities: list[LocalEntityNode] = [] + for node in nodes: + custom_labels = [label for label in node.labels or [] if label not in {"Entity", "Node"}] + if not custom_labels: + continue + if defined_entity_types: + matching_labels = [label for label in custom_labels if label in allowed] + if not matching_labels: + continue + entity_type = matching_labels[0] + else: + entity_type = custom_labels[0] + entity_types.add(entity_type) + entities.append(self._entity(node, enrich_with_edges=enrich_with_edges)) + return LocalFilteredEntities( + entities=entities, + entity_types=entity_types, + total_count=len(nodes), + filtered_count=len(entities), + ) + + def get_entity_with_context( + self, + graph_id_or_entity_uuid: str | None = None, + entity_uuid: str | None = None, + *, + graph_id: str | None = None, + ) -> Optional[LocalEntityNode]: + requested_graph_id = graph_id + if entity_uuid is not None and requested_graph_id is None: + requested_graph_id = graph_id_or_entity_uuid + self._validate_graph_id(requested_graph_id) + node_id = entity_uuid or graph_id_or_entity_uuid + if not node_id: + raise ValueError("memory_entity_id_required") + node = self.repository.get_node(node_id) + return self._entity(node, enrich_with_edges=True) if node is not None else None + + def get_entities_by_type( + self, + graph_id_or_entity_type: str | None = None, + entity_type: str | None = None, + *, + graph_id: str | None = None, + enrich_with_edges: bool = True, + ) -> list[LocalEntityNode]: + requested_graph_id = graph_id + if entity_type is not None and requested_graph_id is None: + requested_graph_id = graph_id_or_entity_type + self._validate_graph_id(requested_graph_id) + selected_type = entity_type or graph_id_or_entity_type + if not selected_type: + raise ValueError("memory_entity_type_required") + return [ + self._entity(node, enrich_with_edges=enrich_with_edges) + for node in self.repository.list_nodes() + if selected_type in (node.labels or []) + ] + + +def make_local_entity_reader_factory(session_factory, *, organization_id: str): + """Create per-worker readers; each reader owns and closes its own session.""" + if not isinstance(organization_id, str) or not organization_id.strip(): + raise ValueError("memory_reader_organization_required") + + def factory(graph_id: str) -> LocalEntityReader: + return LocalEntityReader( + session_factory(), + organization_id=organization_id, + graph_id=graph_id, + owns_session=True, + ) + + return factory diff --git a/backend/app/services/memory_extraction.py b/backend/app/services/memory_extraction.py new file mode 100644 index 0000000..4ea2382 --- /dev/null +++ b/backend/app/services/memory_extraction.py @@ -0,0 +1,143 @@ +"""Strict LLM contract for extracting local graph memory.""" + +from __future__ import annotations + +import json +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field, ValidationError + + +MAX_EPISODE_CHARS = 20_000 +MAX_CONTEXT_CHARS = 12_000 + + +class _StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class ExtractedEntity(_StrictModel): + mention: str = Field(min_length=1, max_length=2_000) + canonical_name: str = Field(min_length=1, max_length=512) + labels: list[str] = Field(default_factory=list, max_length=32) + aliases: list[str] = Field(default_factory=list, max_length=32) + attributes: dict[str, Any] = Field(default_factory=dict) + summary: str = Field(default="", max_length=4_000) + confidence: float = Field(default=0.0, ge=0.0, le=1.0) + + +class ExtractedEdge(_StrictModel): + source_entity_ref: str = Field(min_length=1, max_length=512) + target_entity_ref: str = Field(min_length=1, max_length=512) + relation: str = Field(min_length=1, max_length=128) + fact: str = Field(min_length=1, max_length=4_000) + attributes: dict[str, Any] = Field(default_factory=dict) + valid_at: str | None = Field(default=None, max_length=128) + invalid_at: str | None = Field(default=None, max_length=128) + expired_at: str | None = Field(default=None, max_length=128) + confidence: float = Field(default=0.0, ge=0.0, le=1.0) + evidence: list[str] = Field(default_factory=list, max_length=32) + + +class MemoryExtractionResult(_StrictModel): + entities: list[ExtractedEntity] = Field(default_factory=list, max_length=500) + edges: list[ExtractedEdge] = Field(default_factory=list, max_length=1_000) + episode_summary: str = Field(default="", max_length=8_000) + unresolved_mentions: list[str] = Field(default_factory=list, max_length=200) + + +def parse_extraction_response(raw: str | dict[str, Any]) -> MemoryExtractionResult: + """Parse and validate an LLM response without accepting extra fields.""" + if isinstance(raw, str): + text = raw.strip() + if text.startswith("```"): + lines = text.splitlines() + if lines and lines[0].strip().startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + text = "\n".join(lines).strip() + try: + raw = json.loads(text) + except json.JSONDecodeError as exc: + raise ValueError("invalid_memory_json") from exc + if not isinstance(raw, dict): + raise ValueError("invalid_memory_payload") + return MemoryExtractionResult.model_validate(raw) + + +def _language_instruction(language: str) -> str: + if language == "en": + return "IMPORTANT: Write summaries and facts in English. Return JSON only." + return "IMPORTANT: Write summaries and facts in Thai. Return JSON only." + + +def build_extraction_prompt( + *, + language: str, + ontology: dict[str, Any], + episode_text: str, + context: str = "", +) -> str: + if not isinstance(episode_text, str) or len(episode_text) > MAX_EPISODE_CHARS: + raise ValueError("episode_too_large") + if not isinstance(context, str) or len(context) > MAX_CONTEXT_CHARS: + raise ValueError("context_too_large") + if not isinstance(ontology, dict): + raise ValueError("invalid_ontology") + + ontology_json = json.dumps(ontology, ensure_ascii=False, sort_keys=True) + context_block = context if context else "(none)" + return f"""{_language_instruction(language)} + +You extract evidence-grounded graph memory from one episode. +Never invent facts. Use only labels and relations allowed by the ontology. +Use stable entity_refs inside this response; never guess database IDs. +Preserve temporal fields as null when the evidence does not support a date. +Keep confidence between 0 and 1 and keep evidence references when available. + +Return JSON only with this shape: +{{ + "entities": [{{ + "mention": "text span", + "canonical_name": "stable name", + "labels": ["Person"], + "aliases": [], + "attributes": {{}}, + "summary": "short evidence-grounded summary", + "confidence": 0.0 + }}], + "edges": [{{ + "source_entity_ref": "entity-ref", + "target_entity_ref": "entity-ref", + "relation": "RELATION_NAME", + "fact": "evidence-grounded fact", + "attributes": {{}}, + "valid_at": null, + "invalid_at": null, + "expired_at": null, + "confidence": 0.0, + "evidence": ["episode reference or span"] + }}], + "episode_summary": "short summary", + "unresolved_mentions": [] +}} + +Ontology: +{ontology_json} + +Additional context: +{context_block} + +Episode: +{episode_text} +""" + + +__all__ = [ + "ExtractedEdge", + "ExtractedEntity", + "MemoryExtractionResult", + "build_extraction_prompt", + "parse_extraction_response", +] diff --git a/backend/app/services/memory_repository.py b/backend/app/services/memory_repository.py new file mode 100644 index 0000000..1216c4e --- /dev/null +++ b/backend/app/services/memory_repository.py @@ -0,0 +1,310 @@ +"""Tenant-scoped SQLAlchemy repository for local graph memory.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import Any + +from sqlalchemy import or_, select +from sqlalchemy.orm import Session + +from ..models.memory import MemoryEdge, MemoryEpisode, MemoryGraph, MemoryNode + + +@dataclass(frozen=True) +class MemorySearchResult: + nodes: list[MemoryNode] = field(default_factory=list) + edges: list[MemoryEdge] = field(default_factory=list) + query: str = "" + + @property + def total_count(self) -> int: + return len(self.nodes) + len(self.edges) + + +class SqlAlchemyMemoryRepository: + """All reads and writes are constrained by organization_id + graph_id.""" + + def __init__(self, session: Session, *, organization_id: str, graph_id: str): + if not isinstance(organization_id, str) or not organization_id.strip(): + raise ValueError("invalid_organization_id") + if not isinstance(graph_id, str) or not graph_id.strip(): + raise ValueError("invalid_graph_id") + self.session = session + self.organization_id = organization_id + self.graph_id = graph_id + + def _graph(self) -> MemoryGraph | None: + return self.session.scalar( + select(MemoryGraph).where( + MemoryGraph.id == self.graph_id, + MemoryGraph.organization_id == self.organization_id, + ) + ) + + def _require_graph(self) -> MemoryGraph: + graph = self._graph() + if graph is None: + raise ValueError("memory_graph_not_found") + return graph + + def get_graph(self) -> MemoryGraph: + """Return the graph only when it belongs to this organization scope.""" + return self._require_graph() + + def update_graph_ontology(self, ontology: dict[str, Any]) -> MemoryGraph: + if not isinstance(ontology, dict): + raise ValueError("invalid_ontology") + graph = self._require_graph() + graph.ontology = dict(ontology) + graph.version = int(graph.version or 0) + 1 + self.session.flush() + return graph + + def get_episode(self, *, source_type: str, source_ref: str) -> MemoryEpisode | None: + self._require_graph() + return self.session.scalar( + select(MemoryEpisode).where( + MemoryEpisode.graph_id == self.graph_id, + MemoryEpisode.source_type == source_type, + MemoryEpisode.source_ref == source_ref, + ) + ) + + @staticmethod + def _normalize_name(name: str) -> str: + if not isinstance(name, str) or not name.strip(): + raise ValueError("invalid_memory_name") + return " ".join(name.casefold().split()) + + @staticmethod + def _confidence(value: float) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): + raise ValueError("invalid_confidence") + return min(max(float(value), 0.0), 1.0) + + def create_graph(self, *, project_id: str, ontology: dict[str, Any] | None = None) -> MemoryGraph: + if not isinstance(project_id, str) or not project_id.strip(): + raise ValueError("invalid_project_id") + existing = self.session.get(MemoryGraph, self.graph_id) + if existing is not None: + if existing.organization_id != self.organization_id: + raise ValueError("memory_graph_scope_conflict") + return existing + graph = MemoryGraph( + id=self.graph_id, + organization_id=self.organization_id, + project_id=project_id, + ontology=ontology or {}, + ) + self.session.add(graph) + self.session.flush() + return graph + + def add_episode( + self, + *, + source_type: str, + source_ref: str, + normalized_text: str, + summary: str = "", + extractor_version: str = "v1", + ) -> MemoryEpisode: + self._require_graph() + if not all(isinstance(value, str) and value.strip() for value in (source_type, source_ref, normalized_text)): + raise ValueError("invalid_memory_episode") + episode = self.session.scalar( + select(MemoryEpisode).where( + MemoryEpisode.graph_id == self.graph_id, + MemoryEpisode.source_type == source_type, + MemoryEpisode.source_ref == source_ref, + ) + ) + if episode is None: + episode = MemoryEpisode( + graph_id=self.graph_id, + source_type=source_type, + source_ref=source_ref, + normalized_text=normalized_text, + summary=summary or "", + extractor_version=extractor_version or "v1", + ) + self.session.add(episode) + else: + episode.normalized_text = normalized_text + episode.summary = summary or "" + episode.extractor_version = extractor_version or "v1" + self.session.flush() + return episode + + def upsert_node( + self, + *, + canonical_name: str, + labels: list[str] | None = None, + aliases: list[str] | None = None, + attributes: dict[str, Any] | None = None, + summary: str = "", + confidence: float = 0.0, + ) -> MemoryNode: + self._require_graph() + normalized_name = self._normalize_name(canonical_name) + node = self.session.scalar( + select(MemoryNode).where( + MemoryNode.graph_id == self.graph_id, + MemoryNode.normalized_name == normalized_name, + ) + ) + if node is None: + node = MemoryNode(graph_id=self.graph_id, canonical_name=canonical_name.strip(), normalized_name=normalized_name) + self.session.add(node) + node.canonical_name = canonical_name.strip() + node.labels = list(labels or []) + node.aliases = list(aliases or []) + node.attributes = dict(attributes or {}) + node.summary = summary or "" + node.confidence = self._confidence(confidence) + self.session.flush() + return node + + def upsert_edge( + self, + *, + source_node_id: str, + target_node_id: str, + relation: str, + fact: str, + attributes: dict[str, Any] | None = None, + confidence: float = 0.0, + valid_at=None, + invalid_at=None, + expired_at=None, + ) -> MemoryEdge: + self._require_graph() + if not all(isinstance(value, str) and value.strip() for value in (source_node_id, target_node_id, relation, fact)): + raise ValueError("invalid_memory_edge") + nodes = list( + self.session.scalars( + select(MemoryNode).where( + MemoryNode.graph_id == self.graph_id, + MemoryNode.id.in_([source_node_id, target_node_id]), + ) + ) + ) + if {node.id for node in nodes} != {source_node_id, target_node_id}: + raise ValueError("memory_edge_node_scope_conflict") + edge = self.session.scalar( + select(MemoryEdge).where( + MemoryEdge.graph_id == self.graph_id, + MemoryEdge.source_node_id == source_node_id, + MemoryEdge.target_node_id == target_node_id, + MemoryEdge.relation == relation.strip(), + MemoryEdge.fact == fact.strip(), + ) + ) + if edge is None: + edge = MemoryEdge( + graph_id=self.graph_id, + source_node_id=source_node_id, + target_node_id=target_node_id, + relation=relation.strip(), + fact=fact.strip(), + ) + self.session.add(edge) + edge.attributes = dict(attributes or {}) + edge.confidence = self._confidence(confidence) + edge.valid_at = valid_at + edge.invalid_at = invalid_at + edge.expired_at = expired_at + self.session.flush() + return edge + + def get_node(self, node_id: str) -> MemoryNode | None: + return self.session.scalar( + select(MemoryNode) + .join(MemoryGraph, MemoryGraph.id == MemoryNode.graph_id) + .where( + MemoryNode.id == node_id, + MemoryNode.graph_id == self.graph_id, + MemoryGraph.organization_id == self.organization_id, + ) + ) + + def list_nodes(self, *, limit: int = 1_000) -> list[MemoryNode]: + self._require_graph() + safe_limit = min(max(int(limit), 1), 10_000) + return list( + self.session.scalars( + select(MemoryNode) + .join(MemoryGraph, MemoryGraph.id == MemoryNode.graph_id) + .where( + MemoryNode.graph_id == self.graph_id, + MemoryGraph.organization_id == self.organization_id, + ) + .order_by(MemoryNode.normalized_name.asc(), MemoryNode.id.asc()) + .limit(safe_limit) + ) + ) + + def list_edges(self, *, limit: int = 2_000) -> list[MemoryEdge]: + self._require_graph() + safe_limit = min(max(int(limit), 1), 20_000) + return list( + self.session.scalars( + select(MemoryEdge) + .join(MemoryGraph, MemoryGraph.id == MemoryEdge.graph_id) + .where( + MemoryEdge.graph_id == self.graph_id, + MemoryGraph.organization_id == self.organization_id, + ) + .order_by(MemoryEdge.id.asc()) + .limit(safe_limit) + ) + ) + + def get_node_edges(self, node_id: str, *, limit: int = 500) -> list[MemoryEdge]: + self._require_graph() + safe_limit = min(max(int(limit), 1), 5_000) + return list( + self.session.scalars( + select(MemoryEdge) + .join(MemoryGraph, MemoryGraph.id == MemoryEdge.graph_id) + .where( + MemoryEdge.graph_id == self.graph_id, + MemoryGraph.organization_id == self.organization_id, + (MemoryEdge.source_node_id == node_id) | (MemoryEdge.target_node_id == node_id), + ) + .order_by(MemoryEdge.id.asc()) + .limit(safe_limit) + ) + ) + + def get_node_by_id(self, node_id: str) -> MemoryNode | None: + return self.get_node(node_id) + + def search(self, query: str, *, limit: int = 50) -> MemorySearchResult: + if not isinstance(query, str): + raise ValueError("invalid_memory_query") + normalized_query = " ".join(query.casefold().split()) + if not normalized_query: + return MemorySearchResult(query=query) + safe_limit = min(max(int(limit), 1), 100) + pattern = f"%{normalized_query}%" + nodes = list( + self.session.scalars( + select(MemoryNode) + .join(MemoryGraph, MemoryGraph.id == MemoryNode.graph_id) + .where( + MemoryNode.graph_id == self.graph_id, + MemoryGraph.organization_id == self.organization_id, + or_( + MemoryNode.normalized_name.ilike(pattern), + MemoryNode.summary.ilike(pattern), + ), + ) + .order_by(MemoryNode.normalized_name.asc(), MemoryNode.id.asc()) + .limit(safe_limit) + ) + ) + return MemorySearchResult(nodes=nodes, query=query) diff --git a/backend/app/services/memory_service.py b/backend/app/services/memory_service.py new file mode 100644 index 0000000..cf4000d --- /dev/null +++ b/backend/app/services/memory_service.py @@ -0,0 +1,122 @@ +"""LLM extraction orchestration without giving the model storage authority.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Protocol + +from .memory_extraction import MemoryExtractionResult, build_extraction_prompt, parse_extraction_response +from .memory_repository import SqlAlchemyMemoryRepository + + +class JsonLLMClient(Protocol): + def chat_json(self, messages: list[dict[str, str]], temperature: float = 0.3, max_tokens: int = 4096) -> dict[str, Any]: + ... + + +@dataclass(frozen=True) +class MemoryIngestResult: + entity_count: int + edge_count: int + unresolved_edge_refs: list[str] = field(default_factory=list) + + +class MemoryExtractionService: + def __init__(self, client: JsonLLMClient): + self.client = client + + def extract( + self, + *, + language: str, + ontology: dict[str, Any], + episode_text: str, + context: str = "", + ) -> MemoryExtractionResult: + prompt = build_extraction_prompt( + language=language, + ontology=ontology, + episode_text=episode_text, + context=context, + ) + raw = self.client.chat_json( + [{"role": "system", "content": prompt}], + temperature=0.2, + max_tokens=8192, + ) + return parse_extraction_response(raw) + + @staticmethod + def _ref(value: str) -> str: + return " ".join(value.casefold().split()) + + @staticmethod + def _timestamp(value: str | None): + if value is None: + return None + if not isinstance(value, str) or not value.strip(): + raise ValueError("invalid_memory_timestamp") + try: + parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError("invalid_memory_timestamp") from exc + return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc) + + def persist( + self, + repository: SqlAlchemyMemoryRepository, + result: MemoryExtractionResult, + *, + source_type: str, + source_ref: str, + episode_text: str, + ) -> MemoryIngestResult: + repository.add_episode( + source_type=source_type, + source_ref=source_ref, + normalized_text=episode_text, + summary=result.episode_summary, + ) + + entity_refs: dict[str, str] = {} + entity_count = 0 + for entity in result.entities: + node = repository.upsert_node( + canonical_name=entity.canonical_name, + labels=entity.labels, + aliases=entity.aliases, + attributes=entity.attributes, + summary=entity.summary, + confidence=entity.confidence, + ) + entity_count += 1 + for reference in [entity.mention, entity.canonical_name, *entity.aliases]: + entity_refs[self._ref(reference)] = node.id + + unresolved: list[str] = [] + edge_count = 0 + for edge in result.edges: + source_id = entity_refs.get(self._ref(edge.source_entity_ref)) + target_id = entity_refs.get(self._ref(edge.target_entity_ref)) + if source_id is None or target_id is None: + unresolved.append(f"{edge.source_entity_ref}->{edge.target_entity_ref}") + continue + repository.upsert_edge( + source_node_id=source_id, + target_node_id=target_id, + relation=edge.relation, + fact=edge.fact, + attributes=edge.attributes, + confidence=edge.confidence, + valid_at=self._timestamp(edge.valid_at), + invalid_at=self._timestamp(edge.invalid_at), + expired_at=self._timestamp(edge.expired_at), + ) + edge_count += 1 + + return MemoryIngestResult( + entity_count=entity_count, + edge_count=edge_count, + unresolved_edge_refs=unresolved, + ) diff --git a/backend/app/services/memory_tools.py b/backend/app/services/memory_tools.py new file mode 100644 index 0000000..a4b99cf --- /dev/null +++ b/backend/app/services/memory_tools.py @@ -0,0 +1,506 @@ +"""Local compatibility tools for the legacy ZepToolsService result contract.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Optional + +from .memory_repository import SqlAlchemyMemoryRepository + + +@dataclass +class LocalSearchResult: + facts: list[str] + edges: list[dict[str, Any]] + nodes: list[dict[str, Any]] + query: str + total_count: int + + def to_dict(self) -> dict[str, Any]: + return { + "facts": self.facts, + "edges": self.edges, + "nodes": self.nodes, + "query": self.query, + "total_count": self.total_count, + } + + def to_text(self) -> str: + parts = [f"Search query: {self.query}", f"Found {self.total_count} relevant items"] + if self.facts: + parts.append("\n### Relevant facts:") + parts.extend(f"{index}. {fact}" for index, fact in enumerate(self.facts, 1)) + return "\n".join(parts) + + +@dataclass +class LocalNodeInfo: + uuid: str + name: str + labels: list[str] + summary: str + attributes: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + return { + "uuid": self.uuid, + "name": self.name, + "labels": self.labels, + "summary": self.summary, + "attributes": self.attributes, + } + + +@dataclass +class LocalInsightForgeResult: + query: str + simulation_requirement: str + sub_queries: list[str] + semantic_facts: list[str] = field(default_factory=list) + entity_insights: list[dict[str, Any]] = field(default_factory=list) + relationship_chains: list[str] = field(default_factory=list) + total_facts: int = 0 + total_entities: int = 0 + total_relationships: int = 0 + + def to_text(self) -> str: + parts = [ + "## Local Memory Deep Analysis", + f"Analysis question: {self.query}", + f"Prediction scenario: {self.simulation_requirement}", + f"\n### Statistics\n- Facts: {self.total_facts}\n- Entities: {self.total_entities}\n- Relationships: {self.total_relationships}", + ] + if self.semantic_facts: + parts.append("\n### Key facts\n" + "\n".join(f"{i}. {fact}" for i, fact in enumerate(self.semantic_facts, 1))) + if self.entity_insights: + parts.append( + "\n### Entities\n" + + "\n".join( + f"- {item.get('name', 'Unknown')} ({item.get('type', 'Entity')}): {item.get('summary', '')}" + for item in self.entity_insights + ) + ) + if self.relationship_chains: + parts.append("\n### Relationships\n" + "\n".join(f"- {chain}" for chain in self.relationship_chains)) + return "\n".join(parts) + + +@dataclass +class LocalPanoramaResult: + query: str + all_nodes: list[LocalNodeInfo] = field(default_factory=list) + all_edges: list[LocalEdgeInfo] = field(default_factory=list) + active_facts: list[str] = field(default_factory=list) + historical_facts: list[str] = field(default_factory=list) + + def to_text(self) -> str: + parts = [ + "## Local Memory Panorama", + f"Query: {self.query}", + f"\n### Statistics\n- Total nodes: {len(self.all_nodes)}\n- Total edges: {len(self.all_edges)}\n- Active facts: {len(self.active_facts)}\n- Historical facts: {len(self.historical_facts)}", + ] + if self.active_facts: + parts.append("\n### Active facts\n" + "\n".join(f"{i}. {fact}" for i, fact in enumerate(self.active_facts, 1))) + if self.historical_facts: + parts.append("\n### Historical facts\n" + "\n".join(f"{i}. {fact}" for i, fact in enumerate(self.historical_facts, 1))) + if self.all_nodes: + parts.append("\n### Entities\n" + "\n".join(f"- {node.name}: {node.summary}" for node in self.all_nodes)) + return "\n".join(parts) + + +@dataclass +class LocalInterviewResult: + interview_topic: str + summary: str + + def to_text(self) -> str: + return ( + "## Local Memory Interview\n" + f"Topic: {self.interview_topic}\n\n" + f"{self.summary}" + ) + + +@dataclass +class LocalEdgeInfo: + uuid: str + name: str + fact: str + source_node_uuid: str + target_node_uuid: str + source_node_name: Optional[str] = None + target_node_name: Optional[str] = None + created_at: Optional[str] = None + valid_at: Optional[str] = None + invalid_at: Optional[str] = None + expired_at: Optional[str] = None + + def to_dict(self) -> dict[str, Any]: + return { + "uuid": self.uuid, + "name": self.name, + "fact": self.fact, + "source_node_uuid": self.source_node_uuid, + "target_node_uuid": self.target_node_uuid, + "source_node_name": self.source_node_name, + "target_node_name": self.target_node_name, + "created_at": self.created_at, + "valid_at": self.valid_at, + "invalid_at": self.invalid_at, + "expired_at": self.expired_at, + } + + @property + def is_expired(self) -> bool: + return self.expired_at is not None + + @property + def is_invalid(self) -> bool: + return self.invalid_at is not None + + +class LocalMemoryTools: + def __init__(self, session_or_repository, *, organization_id: str | None = None, graph_id: str | None = None): + if isinstance(session_or_repository, SqlAlchemyMemoryRepository): + self.repository = session_or_repository + else: + if organization_id is None or graph_id is None: + raise ValueError("memory_tools_scope_required") + self.repository = SqlAlchemyMemoryRepository( + session_or_repository, + organization_id=organization_id, + graph_id=graph_id, + ) + + def _validate_graph_id(self, graph_id: str | None = None) -> None: + if graph_id is not None and graph_id != self.repository.graph_id: + raise ValueError("memory_graph_scope_conflict") + + @staticmethod + def _node_info(node) -> LocalNodeInfo: + return LocalNodeInfo( + uuid=node.id, + name=node.canonical_name, + labels=list(node.labels or []), + summary=node.summary or "", + attributes=dict(node.attributes or {}), + ) + + def _edge_info(self, edge, nodes_by_id: dict[str, Any]) -> LocalEdgeInfo: + source = nodes_by_id.get(edge.source_node_id) + target = nodes_by_id.get(edge.target_node_id) + return LocalEdgeInfo( + uuid=edge.id, + name=edge.relation, + fact=edge.fact, + source_node_uuid=edge.source_node_id, + target_node_uuid=edge.target_node_id, + source_node_name=source.canonical_name if source else None, + target_node_name=target.canonical_name if target else None, + created_at=edge.created_at.isoformat() if edge.created_at else None, + valid_at=edge.valid_at.isoformat() if edge.valid_at else None, + invalid_at=edge.invalid_at.isoformat() if edge.invalid_at else None, + expired_at=edge.expired_at.isoformat() if edge.expired_at else None, + ) + + def get_all_nodes(self, *, limit: int = 10_000) -> list[LocalNodeInfo]: + return [self._node_info(node) for node in self.repository.list_nodes(limit=limit)] + + def get_all_edges(self, *, limit: int = 20_000) -> list[LocalEdgeInfo]: + nodes = {node.id: node for node in self.repository.list_nodes(limit=10_000)} + return [self._edge_info(edge, nodes) for edge in self.repository.list_edges(limit=limit)] + + @staticmethod + def _score(query: str, *values: str) -> int: + normalized_query = query.casefold().strip() + haystack = " ".join(value or "" for value in values).casefold() + if not normalized_query or not haystack: + return 0 + if normalized_query in haystack: + return 100 + return sum(10 for token in normalized_query.split() if token in haystack) + + def search_graph( + self, + query: str, + *, + limit: int = 10, + scope: str = "edges", + graph_id: str | None = None, + ) -> LocalSearchResult: + self._validate_graph_id(graph_id) + if scope not in {"edges", "nodes", "both"}: + raise ValueError("invalid_memory_search_scope") + safe_limit = min(max(int(limit), 1), 100) + facts: list[str] = [] + edges: list[dict[str, Any]] = [] + nodes: list[dict[str, Any]] = [] + all_nodes = {node.id: node for node in self.repository.list_nodes(limit=10_000)} + + if scope in {"edges", "both"}: + scored_edges = [ + (self._score(query, edge.fact, edge.relation), edge) + for edge in self.repository.list_edges(limit=20_000) + ] + for score, edge in sorted(scored_edges, key=lambda item: (-item[0], item[1].id))[:safe_limit]: + if score <= 0: + continue + edge_info = self._edge_info(edge, all_nodes).to_dict() + edges.append(edge_info) + if edge.fact: + facts.append(edge.fact) + + if scope in {"nodes", "both"}: + scored_nodes = [ + (self._score(query, node.canonical_name, node.summary), node) + for node in all_nodes.values() + ] + for score, node in sorted(scored_nodes, key=lambda item: (-item[0], item[1].id))[:safe_limit]: + if score <= 0: + continue + node_info = self._node_info(node).to_dict() + node_info.pop("attributes", None) + nodes.append(node_info) + if node.summary: + facts.append(f"[{node.canonical_name}]: {node.summary}") + + return LocalSearchResult( + facts=facts, + edges=edges, + nodes=nodes, + query=query, + total_count=len(facts), + ) + + def quick_search( + self, + query: str, + *, + limit: int = 10, + graph_id: str | None = None, + ) -> LocalSearchResult: + return self.search_graph(query, limit=limit, scope="edges", graph_id=graph_id) + + def get_node_detail(self, node_uuid: str) -> LocalNodeInfo | None: + node = self.repository.get_node(node_uuid) + return self._node_info(node) if node is not None else None + + def get_node_edges(self, node_uuid: str, *, limit: int = 500) -> list[LocalEdgeInfo]: + nodes = {node.id: node for node in self.repository.list_nodes(limit=10_000)} + return [self._edge_info(edge, nodes) for edge in self.repository.get_node_edges(node_uuid, limit=limit)] + + def insight_forge( + self, + *, + graph_id: str, + query: str, + simulation_requirement: str = "", + report_context: str = "", + ) -> LocalInsightForgeResult: + self._validate_graph_id(graph_id) + result = self.search_graph(query, limit=100, scope="edges", graph_id=graph_id) + facts = list(dict.fromkeys(result.facts)) + nodes_by_id = { + node.id: node for node in self.repository.list_nodes(limit=10_000) + } + related_node_ids = list( + dict.fromkeys( + node_id + for edge in result.edges + for node_id in (edge.get("source_node_uuid"), edge.get("target_node_uuid")) + if node_id + ) + ) + entity_insights = [] + for node_id in related_node_ids: + node = nodes_by_id.get(node_id) + if node is None: + continue + entity_insights.append( + { + "uuid": node.id, + "name": node.canonical_name, + "type": next( + (label for label in (node.labels or []) if label not in {"Entity", "Node"}), + "Entity", + ), + "summary": node.summary or "", + "related_facts": [ + fact for fact in facts if node.canonical_name.casefold() in fact.casefold() + ], + } + ) + + relationship_chains = [] + seen_chains = set() + for edge in result.edges: + chain = ( + f"{edge.get('source_node_name') or edge.get('source_node_uuid')} " + f"--[{edge.get('name', '')}]--> " + f"{edge.get('target_node_name') or edge.get('target_node_uuid')}" + ) + if chain not in seen_chains: + seen_chains.add(chain) + relationship_chains.append(chain) + return LocalInsightForgeResult( + query=query, + simulation_requirement=simulation_requirement, + sub_queries=[query] if query else [], + semantic_facts=facts, + entity_insights=entity_insights, + relationship_chains=relationship_chains, + total_facts=len(facts), + total_entities=len(entity_insights), + total_relationships=len(relationship_chains), + ) + + def panorama_search( + self, + *, + graph_id: str, + query: str, + include_expired: bool = True, + limit: int = 50, + ) -> LocalPanoramaResult: + self._validate_graph_id(graph_id) + safe_limit = min(max(int(limit), 1), 100) + nodes = self.get_all_nodes(limit=10_000) + edges = self.get_all_edges(limit=20_000) + + active_facts = [] + historical_facts = [] + for edge in edges: + if not edge.fact: + continue + if edge.is_invalid or edge.is_expired: + valid_at = edge.valid_at or "Unknown" + invalid_at = edge.invalid_at or edge.expired_at or "Unknown" + historical_facts.append(f"[{valid_at} - {invalid_at}] {edge.fact}") + else: + active_facts.append(edge.fact) + + query_lower = query.casefold() + keywords = [ + word.strip() + for word in query_lower.replace(",", " ").replace("，", " ").split() + if len(word.strip()) > 1 + ] + + def relevance_score(fact: str) -> int: + fact_lower = fact.casefold() + score = 100 if query_lower in fact_lower else 0 + return score + sum(10 for keyword in keywords if keyword in fact_lower) + + active_facts.sort(key=relevance_score, reverse=True) + historical_facts.sort(key=relevance_score, reverse=True) + return LocalPanoramaResult( + query=query, + all_nodes=nodes, + all_edges=edges, + active_facts=active_facts[:safe_limit], + historical_facts=historical_facts[:safe_limit] if include_expired else [], + ) + + def get_entity_summary(self, *, graph_id: str, entity_name: str) -> dict[str, Any]: + self._validate_graph_id(graph_id) + normalized = " ".join((entity_name or "").casefold().split()) + node = next( + ( + item + for item in self.repository.list_nodes(limit=10_000) + if item.normalized_name == normalized + ), + None, + ) + if node is None: + return {"entity_name": entity_name, "found": False, "summary": "", "related_facts": []} + edges = self.get_node_edges(node.id, limit=500) + return { + "entity_name": node.canonical_name, + "found": True, + "uuid": node.id, + "labels": list(node.labels or []), + "summary": node.summary or "", + "attributes": dict(node.attributes or {}), + "related_facts": [edge.fact for edge in edges], + } + + def get_entities_by_type(self, *, graph_id: str, entity_type: str) -> list[LocalNodeInfo]: + self._validate_graph_id(graph_id) + target = (entity_type or "").casefold().strip() + return [ + self._node_info(node) + for node in self.repository.list_nodes(limit=10_000) + if target in {str(label).casefold() for label in (node.labels or [])} + ] + + def interview_agents( + self, + *, + simulation_id: str, + interview_requirement: str, + simulation_requirement: str, + max_agents: int = 5, + ) -> LocalInterviewResult: + del simulation_id, max_agents + return LocalInterviewResult( + interview_topic=interview_requirement, + summary=( + "Local memory stores graph facts, not simulation transcripts. " + "Use the retrieved entities and relationships as evidence; no synthetic interview response was generated." + ), + ) + + def get_simulation_context( + self, + *, + graph_id: str, + simulation_requirement: str, + limit: int = 30, + ) -> dict[str, Any]: + self._validate_graph_id(graph_id) + result = self.search_graph( + simulation_requirement, + limit=limit, + scope="both", + graph_id=graph_id, + ) + nodes = self.repository.list_nodes(limit=10_000) + entities = [ + { + "name": node.canonical_name, + "type": next((label for label in (node.labels or []) if label not in {"Entity", "Node"}), "Entity"), + "summary": node.summary or "", + } + for node in nodes + if any(label not in {"Entity", "Node"} for label in (node.labels or [])) + ] + return { + "simulation_requirement": simulation_requirement, + "related_facts": result.facts, + "graph_statistics": self.get_graph_statistics(graph_id), + "entities": entities[: max(int(limit), 1)], + "total_entities": len(entities), + } + + def get_graph_statistics(self, graph_id: str | None = None) -> dict[str, Any]: + self._validate_graph_id(graph_id) + nodes = self.repository.list_nodes(limit=10_000) + edges = self.repository.list_edges(limit=20_000) + entity_types: dict[str, int] = {} + for node in nodes: + for label in node.labels or []: + if label not in {"Entity", "Node"}: + entity_types[label] = entity_types.get(label, 0) + 1 + relation_types: dict[str, int] = {} + for edge in edges: + relation_types[edge.relation] = relation_types.get(edge.relation, 0) + 1 + return { + "graph_id": self.repository.graph_id, + "node_count": len(nodes), + "edge_count": len(edges), + "total_nodes": len(nodes), + "total_edges": len(edges), + "entity_types": entity_types, + "relation_types": relation_types, + } diff --git a/backend/app/services/oasis_profile_generator.py b/backend/app/services/oasis_profile_generator.py index e4914a3..7ca4387 100644 --- a/backend/app/services/oasis_profile_generator.py +++ b/backend/app/services/oasis_profile_generator.py @@ -8,6 +8,8 @@ OASIS Agent Profile生成器 3. 区分个人实体和抽象群体实体 """ +from __future__ import annotations + import json import random import time @@ -16,12 +18,13 @@ from dataclasses import dataclass, field from datetime import datetime from openai import OpenAI -from zep_cloud.client import Zep +from typing import TYPE_CHECKING from ..config import Config from ..utils.logger import get_logger from ..utils.locale import get_language_instruction, get_locale, set_locale, t -from .zep_entity_reader import EntityNode, ZepEntityReader +if TYPE_CHECKING: + from .zep_entity_reader import EntityNode logger = get_logger('crowdsight.oasis_profile') @@ -184,7 +187,9 @@ class OasisProfileGenerator: base_url: Optional[str] = None, model_name: Optional[str] = None, zep_api_key: Optional[str] = None, - graph_id: Optional[str] = None + graph_id: Optional[str] = None, + use_zep_context: bool = True, + local_memory_tools: Optional[Any] = None, ): self.api_key = api_key or Config.LLM_API_KEY self.base_url = base_url or Config.LLM_BASE_URL @@ -198,13 +203,18 @@ class OasisProfileGenerator: base_url=self.base_url ) - # Zep客户端用于检索丰富上下文 - self.zep_api_key = zep_api_key or Config.ZEP_API_KEY + # Local mode is a hard boundary: never construct a Zep client, even + # when a legacy Zep key is present in the process environment. + zep_context_enabled = use_zep_context and Config.MEMORY_BACKEND != "local" + self.zep_api_key = (zep_api_key or Config.ZEP_API_KEY) if zep_context_enabled else None self.zep_client = None self.graph_id = graph_id + self.local_memory_tools = local_memory_tools if self.zep_api_key: try: + from zep_cloud.client import Zep + self.zep_client = Zep(api_key=self.zep_api_key) except Exception as e: logger.warning(f"Zep客户端初始化失败: {e}") @@ -285,6 +295,45 @@ class OasisProfileGenerator: suffix = random.randint(100, 999) return f"{username}_{suffix}" + def _search_local_for_entity(self, entity: EntityNode) -> Dict[str, Any]: + """Retrieve bounded deterministic context from the local memory tools.""" + empty = {"facts": [], "node_summaries": [], "context": ""} + if self.local_memory_tools is None: + return empty + try: + result = self.local_memory_tools.search_graph( + entity.name, + limit=30, + scope="both", + ) + facts = list(getattr(result, "facts", []) or []) + node_summaries = [] + for node in list(getattr(result, "nodes", []) or []): + if not isinstance(node, dict): + continue + summary = node.get("summary") + name = node.get("name") + if summary: + node_summaries.append(str(summary)) + elif name and name != entity.name: + node_summaries.append(f"Related entity: {name}") + context_parts = [] + if facts: + context_parts.append("### Local memory facts\n" + "\n".join(f"- {fact}" for fact in facts[:20])) + if node_summaries: + context_parts.append( + "### Local memory related entities\n" + + "\n".join(f"- {summary}" for summary in node_summaries[:10]) + ) + return { + "facts": facts, + "node_summaries": node_summaries, + "context": "\n\n".join(context_parts), + } + except Exception as exc: + logger.warning("Local memory context lookup failed: %s", type(exc).__name__) + return empty + def _search_zep_for_entity(self, entity: EntityNode) -> Dict[str, Any]: """ 使用Zep图谱混合搜索功能获取实体相关的丰富信息 @@ -474,17 +523,28 @@ class OasisProfileGenerator: if related_info: context_parts.append("### 关联实体信息\n" + "\n".join(related_info)) - # 4. 使用Zep混合检索获取更丰富的信息 - zep_results = self._search_zep_for_entity(entity) - - if zep_results.get("facts"): + # Use exactly one deterministic memory backend for enrichment. + if self.local_memory_tools is not None: + memory_results = self._search_local_for_entity(entity) + memory_label = "Local memory" + else: + memory_results = self._search_zep_for_entity(entity) + memory_label = "Zep" + + if memory_results.get("facts"): # 去重：排除已存在的事实 - new_facts = [f for f in zep_results["facts"] if f not in existing_facts] + new_facts = [f for f in memory_results["facts"] if f not in existing_facts] if new_facts: - context_parts.append("### Zep检索到的事实信息\n" + "\n".join(f"- {f}" for f in new_facts[:15])) - - if zep_results.get("node_summaries"): - context_parts.append("### Zep检索到的相关节点\n" + "\n".join(f"- {s}" for s in zep_results["node_summaries"][:10])) + context_parts.append( + f"### {memory_label} retrieved facts\n" + + "\n".join(f"- {f}" for f in new_facts[:15]) + ) + + if memory_results.get("node_summaries"): + context_parts.append( + f"### {memory_label} related nodes\n" + + "\n".join(f"- {s}" for s in memory_results["node_summaries"][:10]) + ) return "\n\n".join(context_parts) diff --git a/backend/app/services/password_reset.py b/backend/app/services/password_reset.py new file mode 100644 index 0000000..6ca5bee --- /dev/null +++ b/backend/app/services/password_reset.py @@ -0,0 +1,76 @@ +"""Durable, single-use password reset token service. + +Tokens are opaque random strings; only their SHA-256 hash is stored. Consuming +a valid token marks it used and bumps the user's ``auth_version`` so previously +issued sessions are invalidated. +""" + +from __future__ import annotations + +import hashlib +import secrets +from datetime import datetime, timedelta, timezone +from typing import Optional + +from sqlalchemy.orm import Session + +from ..models.password_reset import PasswordResetToken +from ..models.saas import User + +DEFAULT_TTL = timedelta(hours=24) +DEFAULT_TTL_HOURS = 24 + + +def _hash(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def _utc_now() -> datetime: + # Use naive UTC: SQLite returns naive datetimes for DateTime(timezone=True) + # on read, so comparisons are kept tz-naive to avoid aware/naive errors. + return datetime.utcnow() + + +class PasswordResetService: + def __init__(self, session: Session, *, ttl: timedelta | None = None): + self.session = session + self.ttl = ttl or DEFAULT_TTL + + def create_token(self, *, user_id: str) -> str: + token = secrets.token_urlsafe(32) + record = PasswordResetToken( + user_id=user_id, + token_hash=_hash(token), + expires_at=_utc_now() + self.ttl, + ) + self.session.add(record) + self.session.flush() + return token + + def _latest_token(self, user_id: str) -> Optional[PasswordResetToken]: + return ( + self.session.query(PasswordResetToken) + .filter(PasswordResetToken.user_id == user_id) + .order_by(PasswordResetToken.created_at.desc()) + .first() + ) + + def consume_token(self, token: str, *, user_id: str) -> bool: + """Consume a valid, unexpired, unused token for ``user_id``. + + On success marks the token used and increments the user's auth_version + (invalidating older sessions). Returns False otherwise. + """ + record = self._latest_token(user_id) + if record is None or record.used: + return False + if _hash(token) != record.token_hash: + return False + if record.expires_at < _utc_now(): + return False + record.used = True + user = self.session.get(User, user_id) + if user is not None: + user.auth_version = (user.auth_version or 0) + 1 + self.session.flush() + return True diff --git a/backend/app/services/product_repository.py b/backend/app/services/product_repository.py new file mode 100644 index 0000000..5aa9950 --- /dev/null +++ b/backend/app/services/product_repository.py @@ -0,0 +1,379 @@ +"""Tenant-scoped repository for durable product resources. + +Projects, simulations, and reports owned by an organization and (for projects) +an owner/creator. Every lookup requires an explicit ``organization_id`` so an +unscoped id supplied by a route can never be read across tenant boundaries. + +This repository is flush-only: the caller owns transaction boundaries and must +commit. Nothing here persists secrets or raw exception details. +""" + +from __future__ import annotations + +from sqlalchemy.orm import Session + + +class ProductRepository: + """Flush-only durable read/write for product resources.""" + + def __init__(self, session: Session): + self.session = session + + # ---- projects ---- + + def create_project( + self, + *, + organization_id: str, + owner_user_id: str | None, + name: str, + language: str = "en", + ontology: dict | None = None, + ): + from app.models.product import ProductProject + + if not isinstance(organization_id, str) or not organization_id: + raise ValueError("organization_id_required") + if not isinstance(name, str): + raise ValueError("invalid_project_name") + project = ProductProject( + organization_id=organization_id, + owner_user_id=owner_user_id, + name=name, + language=language, + ontology=ontology, + ) + self.session.add(project) + self.session.flush() + return project + + def get_project(self, project_id: str, *, organization_id: str): + from app.models.product import ProductProject + + return ( + self.session.query(ProductProject) + .filter( + ProductProject.id == project_id, + ProductProject.organization_id == organization_id, + ) + .first() + ) + + def sync_project(self, state, *, commit: bool = False): + """Mirror a legacy filesystem Project state into durable SQL (idempotent). + + ``state`` may be a ``Project`` object or any object exposing the durable + fields (e.g. ``ProjectManager.to_dict()``). The sync is an upsert keyed on + the project id + organization so repeated writes do not duplicate rows. + """ + from app.models.product import ProductProject, ProjectStatus + + def _get(names, default=None): + for name in names: + value = getattr(state, name, None) + if value is not None: + return value + if isinstance(state, dict): + value = state.get(name, None) + if value is not None: + return value + return default + + project_id = _get(["project_id", "id"]) + organization_id = _get(["organization_id"]) + if not project_id or not organization_id: + raise ValueError("project_sync_requires_id_and_organization") + + row = ( + self.session.query(ProductProject) + .filter( + ProductProject.id == project_id, + ProductProject.organization_id == organization_id, + ) + .first() + ) + if row is None: + row = ProductProject( + id=project_id, + organization_id=organization_id, + owner_user_id=_get(["owner_user_id"]), + name=_get(["name"], "") or "", + status=_get(["status"], ProjectStatus.CREATED.value), + language=_get(["language"], "en"), + total_text_length=int(_get(["total_text_length"], 0) or 0), + source_metadata=_get(["source_metadata"]), + ontology=_get(["ontology"]), + analysis_summary=_get(["analysis_summary"]), + simulation_requirement=_get(["simulation_requirement"]), + graph_id=_get(["graph_id"]), + graph_build_task_id=_get(["graph_build_task_id"]), + error=_get(["error"]), + ) + self.session.add(row) + else: + row.owner_user_id = _get(["owner_user_id"], row.owner_user_id) + row.name = _get(["name"], row.name) or row.name + row.status = _get(["status"], row.status) + row.language = _get(["language"], row.language) + row.total_text_length = int(_get(["total_text_length"], row.total_text_length) or 0) + if (_get(["ontology"])) is not None: + row.ontology = _get(["ontology"]) + if (_get(["analysis_summary"])) is not None: + row.analysis_summary = _get(["analysis_summary"]) + if (_get(["simulation_requirement"])) is not None: + row.simulation_requirement = _get(["simulation_requirement"]) + if (_get(["graph_id"])) is not None: + row.graph_id = _get(["graph_id"]) + if (_get(["graph_build_task_id"])) is not None: + row.graph_build_task_id = _get(["graph_build_task_id"]) + if (_get(["error"])) is not None: + row.error = _get(["error"]) + self.session.flush() + if commit: + self.session.commit() + return row + + def list_projects( + self, *, organization_id: str, owner_user_id: str | None = None, limit: int = 200 + ) -> list: + from app.models.product import ProductProject + + query = self.session.query(ProductProject).filter( + ProductProject.organization_id == organization_id + ) + if owner_user_id is not None: + query = query.filter(ProductProject.owner_user_id == owner_user_id) + return query.order_by(ProductProject.created_at.desc()).limit(limit).all() + + # ---- simulations ---- + + def create_simulation( + self, + *, + organization_id: str, + project_id: str, + created_by_user_id: str | None, + status: str = "created", + platform: str = "parallel", + config: dict | None = None, + ): + from app.models.product import ProductSimulation + + self._require_org(organization_id) + simulation = ProductSimulation( + organization_id=organization_id, + project_id=project_id, + created_by_user_id=created_by_user_id, + status=status, + platform=platform, + config=config, + ) + self.session.add(simulation) + self.session.flush() + return simulation + + def sync_simulation(self, state, *, commit: bool = False): + """Mirror a legacy simulation state into durable SQL (idempotent upsert).""" + from app.models.product import ProductSimulation + + def _get(names, default=None): + for name in names: + value = getattr(state, name, None) + if value is not None: + return value + if isinstance(state, dict): + value = state.get(name, None) + if value is not None: + return value + return default + + simulation_id = _get(["simulation_id", "id"]) + organization_id = _get(["organization_id"]) + project_id = _get(["project_id"]) + if not simulation_id or not organization_id: + raise ValueError("simulation_sync_requires_id_and_organization") + + row = ( + self.session.query(ProductSimulation) + .filter( + ProductSimulation.id == simulation_id, + ProductSimulation.organization_id == organization_id, + ) + .first() + ) + if row is None: + row = ProductSimulation( + id=simulation_id, + organization_id=organization_id, + project_id=project_id or "", + created_by_user_id=_get(["created_by_user_id", "owner_user_id"]), + status=_get(["status"], "created"), + platform=_get(["platform"], "parallel"), + config=_get(["config"]), + current_round=int(_get(["current_round"], 0) or 0), + ) + self.session.add(row) + else: + row.project_id = project_id or row.project_id + row.status = _get(["status"], row.status) + row.platform = _get(["platform"], row.platform) + if (_get(["config"])) is not None: + row.config = _get(["config"]) + row.current_round = int(_get(["current_round"], row.current_round) or 0) + self.session.flush() + if commit: + self.session.commit() + return row + + def get_simulation(self, simulation_id: str, *, organization_id: str): + from app.models.product import ProductSimulation + + return ( + self.session.query(ProductSimulation) + .filter( + ProductSimulation.id == simulation_id, + ProductSimulation.organization_id == organization_id, + ) + .first() + ) + + def list_simulations( + self, *, organization_id: str, project_id: str | None = None, limit: int = 200 + ) -> list: + from app.models.product import ProductSimulation + + query = self.session.query(ProductSimulation).filter( + ProductSimulation.organization_id == organization_id + ) + if project_id is not None: + query = query.filter(ProductSimulation.project_id == project_id) + return query.order_by(ProductSimulation.created_at.desc()).limit(limit).all() + + # ---- reports ---- + + def create_report( + self, + *, + organization_id: str, + project_id: str, + simulation_id: str | None, + created_by_user_id: str | None, + title: str = "", + status: str = "draft", + ): + from app.models.product import DurableReport + + self._require_org(organization_id) + report = DurableReport( + organization_id=organization_id, + project_id=project_id, + simulation_id=simulation_id, + created_by_user_id=created_by_user_id, + title=title, + status=status, + ) + self.session.add(report) + self.session.flush() + return report + + def get_report(self, report_id: str, *, organization_id: str): + from app.models.product import DurableReport + + return ( + self.session.query(DurableReport) + .filter( + DurableReport.id == report_id, + DurableReport.organization_id == organization_id, + ) + .first() + ) + + def sync_report( + self, + state, + *, + organization_id: str, + project_id: str, + simulation_id: str | None, + created_by_user_id: str | None, + commit: bool = False, + ): + """Mirror a legacy report state into durable SQL (idempotent upsert).""" + from app.models.product import DurableReport + + def _get(names, default=None): + for name in names: + value = getattr(state, name, None) + if value is not None: + return value + if isinstance(state, dict): + value = state.get(name, None) + if value is not None: + return value + return default + + report_id = _get(["report_id", "id"]) + if not report_id: + raise ValueError("report_sync_requires_id") + + row = ( + self.session.query(DurableReport) + .filter( + DurableReport.id == report_id, + DurableReport.organization_id == organization_id, + ) + .first() + ) + if row is None: + row = DurableReport( + id=report_id, + organization_id=organization_id, + project_id=project_id or "", + simulation_id=simulation_id, + created_by_user_id=created_by_user_id, + status=_get(["status"], "draft"), + title=_get(["title"], "") or "", + outline=_get(["outline"]), + markdown_content=_get(["markdown_content", "content"]), + error=_get(["error"]), + ) + self.session.add(row) + else: + row.status = _get(["status"], row.status) + _title = _get(["title"]) + if _title is not None: + row.title = _title + if (_get(["outline"])) is not None: + row.outline = _get(["outline"]) + if (_get(["markdown_content", "content"])) is not None: + row.markdown_content = _get(["markdown_content", "content"]) + if (_get(["error"])) is not None: + row.error = _get(["error"]) + self.session.flush() + if commit: + self.session.commit() + return row + + def list_reports( + self, + *, + organization_id: str, + project_id: str | None = None, + simulation_id: str | None = None, + limit: int = 200, + ) -> list: + from app.models.product import DurableReport + + query = self.session.query(DurableReport).filter( + DurableReport.organization_id == organization_id + ) + if project_id is not None: + query = query.filter(DurableReport.project_id == project_id) + if simulation_id is not None: + query = query.filter(DurableReport.simulation_id == simulation_id) + return query.order_by(DurableReport.created_at.desc()).limit(limit).all() + + @staticmethod + def _require_org(organization_id: str) -> None: + if not isinstance(organization_id, str) or not organization_id: + raise ValueError("organization_id_required") diff --git a/backend/app/services/rate_limiter.py b/backend/app/services/rate_limiter.py new file mode 100644 index 0000000..0de2631 --- /dev/null +++ b/backend/app/services/rate_limiter.py @@ -0,0 +1,49 @@ +"""Durable rate limiting (sliding-window counter) over the rate_limit_events table. + +A durable counter table means limits survive worker restarts and multi-instance +deploys. Keys are typically ``:``. No secrets are stored. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +from sqlalchemy.orm import Session + +from ..models.rate_limit import RateLimitEvent + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class RateLimiter: + """Flush-only rate limiter; caller owns the transaction boundary.""" + + def __init__(self, session: Session, *, window: timedelta | None = None, limit: int | None = None): + self.session = session + self.window = window or timedelta(minutes=15) + self.limit = limit or 60 + + def check_and_record(self, operation: str, *, key: str, organization_id: str | None = None) -> bool: + """Record a hit if under the limit; return False when over the limit.""" + cutoff = _utc_now() - self.window + count = self.session.query(RateLimitEvent).filter( + RateLimitEvent.operation == operation, + RateLimitEvent.key == key, + RateLimitEvent.created_at >= cutoff, + ).count() + if count >= self.limit: + return False + self.session.add( + RateLimitEvent( + operation=operation, + key=key, + organization_id=organization_id, + ) + ) + # Rate-limit events are append-only side effects that must survive even + # if the surrounding request transaction is later rolled back, so commit + # them immediately. + self.session.commit() + return True diff --git a/backend/app/services/report_agent.py b/backend/app/services/report_agent.py index 3ab964e..fa00f0c 100644 --- a/backend/app/services/report_agent.py +++ b/backend/app/services/report_agent.py @@ -22,13 +22,6 @@ from ..config import Config from ..utils.llm_client import LLMClient from ..utils.logger import get_logger from ..utils.locale import get_language_instruction, t -from .zep_tools import ( - ZepToolsService, - SearchResult, - InsightForgeResult, - PanoramaResult, - InterviewResult -) logger = get_logger('crowdsight.report_agent') @@ -886,7 +879,8 @@ class ReportAgent: simulation_id: str, simulation_requirement: str, llm_client: Optional[LLMClient] = None, - zep_tools: Optional[ZepToolsService] = None + memory_tools: Optional[Any] = None, + zep_tools: Optional[Any] = None, ): """ 初始化Report Agent @@ -903,7 +897,20 @@ class ReportAgent: self.simulation_requirement = simulation_requirement self.llm = llm_client or LLMClient() - self.zep_tools = zep_tools or ZepToolsService() + if memory_tools is not None: + self.memory_tools = memory_tools + elif Config.MEMORY_BACKEND == "local": + # Local mode must be explicit. Constructing a Zep client here would + # create a silent backend fallback and can cross tenant boundaries. + raise ValueError("memory_tools_required_for_local_backend") + elif zep_tools is not None: + # ``zep_tools`` remains a compatibility injection for the legacy + # backend; it is never considered in local mode. + self.memory_tools = zep_tools + else: + from .zep_tools import ZepToolsService + + self.memory_tools = ZepToolsService() # 工具定义 self.tools = self._define_tools() @@ -970,7 +977,7 @@ class ReportAgent: if tool_name == "insight_forge": query = parameters.get("query", "") ctx = parameters.get("report_context", "") or report_context - result = self.zep_tools.insight_forge( + result = self.memory_tools.insight_forge( graph_id=self.graph_id, query=query, simulation_requirement=self.simulation_requirement, @@ -984,7 +991,7 @@ class ReportAgent: include_expired = parameters.get("include_expired", True) if isinstance(include_expired, str): include_expired = include_expired.lower() in ['true', '1', 'yes'] - result = self.zep_tools.panorama_search( + result = self.memory_tools.panorama_search( graph_id=self.graph_id, query=query, include_expired=include_expired @@ -997,7 +1004,7 @@ class ReportAgent: limit = parameters.get("limit", 10) if isinstance(limit, str): limit = int(limit) - result = self.zep_tools.quick_search( + result = self.memory_tools.quick_search( graph_id=self.graph_id, query=query, limit=limit @@ -1011,7 +1018,7 @@ class ReportAgent: if isinstance(max_agents, str): max_agents = int(max_agents) max_agents = min(max_agents, 10) - result = self.zep_tools.interview_agents( + result = self.memory_tools.interview_agents( simulation_id=self.simulation_id, interview_requirement=interview_topic, simulation_requirement=self.simulation_requirement, @@ -1027,12 +1034,12 @@ class ReportAgent: return self._execute_tool("quick_search", parameters, report_context) elif tool_name == "get_graph_statistics": - result = self.zep_tools.get_graph_statistics(self.graph_id) + result = self.memory_tools.get_graph_statistics(self.graph_id) return json.dumps(result, ensure_ascii=False, indent=2) elif tool_name == "get_entity_summary": entity_name = parameters.get("entity_name", "") - result = self.zep_tools.get_entity_summary( + result = self.memory_tools.get_entity_summary( graph_id=self.graph_id, entity_name=entity_name ) @@ -1046,7 +1053,7 @@ class ReportAgent: elif tool_name == "get_entities_by_type": entity_type = parameters.get("entity_type", "") - nodes = self.zep_tools.get_entities_by_type( + nodes = self.memory_tools.get_entities_by_type( graph_id=self.graph_id, entity_type=entity_type ) @@ -1056,9 +1063,9 @@ class ReportAgent: else: return f"Unknown tool: {tool_name}. Please use one of the following tools: insight_forge, panorama_search, quick_search" - except Exception as e: - logger.error(t('report.toolExecFailed', toolName=tool_name, error=str(e))) - return f"Tool execution failed: {str(e)}" + except Exception as exc: + logger.error("Report tool execution failed: tool=%s error=%s", tool_name, type(exc).__name__) + return "Tool execution failed; no result is available." # 合法的工具名称集合，用于裸 JSON 兜底解析时校验 VALID_TOOL_NAMES = {"insight_forge", "panorama_search", "quick_search", "interview_agents"} @@ -1154,7 +1161,7 @@ class ReportAgent: progress_callback("planning", 0, t('progress.analyzingRequirements')) # 首先获取模拟上下文 - context = self.zep_tools.get_simulation_context( + context = self.memory_tools.get_simulation_context( graph_id=self.graph_id, simulation_requirement=self.simulation_requirement ) @@ -1737,19 +1744,22 @@ class ReportAgent: return report except Exception as e: - logger.error(t('report.reportGenFailed', error=str(e))) + logger.error( + "Report generation failed: error_type=%s", + type(e).__name__, + ) report.status = ReportStatus.FAILED - report.error = str(e) + report.error = t('api.internalError') # 记录错误日志 if self.report_logger: - self.report_logger.log_error(str(e), "failed") + self.report_logger.log_error("report_generation_failed", "failed") # 保存失败状态 try: ReportManager.save_report(report) ReportManager.update_progress( - report_id, "failed", -1, t('progress.reportFailed', error=str(e)), + report_id, "failed", -1, t('api.internalError'), completed_sections=completed_section_titles ) except Exception: @@ -1797,7 +1807,10 @@ class ReportAgent: if len(report.markdown_content) > 15000: report_content += "\n\n... [报告内容已截断] ..." except Exception as e: - logger.warning(t('report.fetchReportFailed', error=e)) + logger.warning( + "Failed to fetch report for chat: error_type=%s", + type(e).__name__, + ) system_prompt = CHAT_SYSTEM_PROMPT_TEMPLATE.format( simulation_requirement=self.simulation_requirement, diff --git a/backend/app/services/settings_service.py b/backend/app/services/settings_service.py new file mode 100644 index 0000000..f94dc55 --- /dev/null +++ b/backend/app/services/settings_service.py @@ -0,0 +1,115 @@ +"""Versioned, redacted platform settings service. + +The API key is encrypted with a Fernet key derived from ``SECRET_KEY`` so the +plaintext never appears in the record, API responses, or logs. The public API +surface only ever sees a masked value and the settings version; a per-job +snapshot references the version rather than the secret. +""" + +from __future__ import annotations + +import base64 +import hashlib +from datetime import datetime, timezone +from typing import Optional +from uuid import uuid4 + +from cryptography.fernet import Fernet +from sqlalchemy.orm import Session + +from ..config import Config +from ..models.settings import PlatformSettings + +MASK = "sk-••••••••" + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class SettingsService: + """Flush-only settings repository; caller owns transactions.""" + + def __init__(self, session: Session): + self.session = session + + def _fernet(self) -> Fernet: + secret = Config.SECRET_KEY + if not secret: + raise ValueError("settings_secret_key_required") + digest = hashlib.sha256(secret.encode("utf-8")).digest() + key = base64.urlsafe_b64encode(digest) + return Fernet(key) + + def _encrypt_secret(self, api_key: str) -> str: + return self._fernet().encrypt(api_key.encode("utf-8")).decode("utf-8") + + def _decrypt_secret(self, secret_ref: str) -> Optional[str]: + if not secret_ref: + return None + try: + return self._fernet().decrypt(secret_ref.encode("utf-8")).decode("utf-8") + except Exception: + return None + + def _latest_row(self) -> Optional[PlatformSettings]: + return ( + self.session.query(PlatformSettings) + .order_by(PlatformSettings.created_at.desc()) + .first() + ) + + def save_settings( + self, + settings: dict, + *, + api_key: Optional[str] = None, + updated_by: Optional[str] = None, + ) -> str: + """Persist a new version, encrypting the API key if provided.""" + version = f"v{uuid4().hex[:12]}" + # Clear active on all existing rows, then insert the new active version. + for row in self.session.query(PlatformSettings).filter( + PlatformSettings.active.is_(True) + ): + row.active = False + record = PlatformSettings( + version=version, + settings=settings, + secret_ref=self._encrypt_secret(api_key) if api_key else None, + updated_by_user_id=updated_by, + active=True, + ) + self.session.add(record) + self.session.flush() + return version + + def _mask(self, settings: dict) -> dict: + out = dict(settings or {}) + # A secret value should never be in the public dict; be defensive. + for key in list(out.keys()): + if "key" in key.lower() or "secret" in key.lower() or "token" in key.lower(): + out[key] = MASK + return out + + def active_settings(self) -> dict: + row = self._latest_row() + if row is None: + return {"settings": {}, "version": None, "updated_by": None, "api_key": MASK} + masked = self._mask(row.settings if isinstance(row.settings, dict) else {}) + return { + "settings": masked, + "version": row.version, + "updated_by": row.updated_by_user_id, + "api_key": MASK if row.secret_ref else MASK, + } + + def snapshot_for_job(self) -> dict: + row = self._latest_row() + if row is None: + return {"settings": {}, "version": None, "settings_version": None} + return { + "settings": self._mask(row.settings if isinstance(row.settings, dict) else {}), + "version": row.version, + "settings_version": row.version, + } diff --git a/backend/app/services/simulation_config_generator.py b/backend/app/services/simulation_config_generator.py index 270bd31..f10a16f 100644 --- a/backend/app/services/simulation_config_generator.py +++ b/backend/app/services/simulation_config_generator.py @@ -10,9 +10,11 @@ 4. 生成平台配置 """ +from __future__ import annotations + import json import math -from typing import Dict, Any, List, Optional, Callable +from typing import Dict, Any, List, Optional, Callable, TYPE_CHECKING from dataclasses import dataclass, field, asdict from datetime import datetime @@ -21,7 +23,8 @@ from openai import OpenAI from ..config import Config from ..utils.logger import get_logger from ..utils.locale import get_language_instruction, t -from .zep_entity_reader import EntityNode, ZepEntityReader +if TYPE_CHECKING: + from .zep_entity_reader import EntityNode logger = get_logger('crowdsight.simulation_config') diff --git a/backend/app/services/simulation_manager.py b/backend/app/services/simulation_manager.py index 1531f5b..f2a3916 100644 --- a/backend/app/services/simulation_manager.py +++ b/backend/app/services/simulation_manager.py @@ -7,15 +7,15 @@ OASIS模拟管理器 import os import json import shutil -from typing import Dict, Any, List, Optional +from typing import Callable, Dict, Any, List, Optional from dataclasses import dataclass, field from datetime import datetime from enum import Enum from ..config import Config from ..utils.logger import get_logger -from .zep_entity_reader import ZepEntityReader, FilteredEntities from .oasis_profile_generator import OasisProfileGenerator, OasisAgentProfile +from .memory_tools import LocalMemoryTools from .simulation_config_generator import SimulationConfigGenerator, SimulationParameters from ..utils.locale import t @@ -47,6 +47,10 @@ class SimulationState: project_id: str graph_id: str + # Tenant/owner scope (populated by the creating route for durable sync). + organization_id: Optional[str] = None + owner_user_id: Optional[str] = None + # 平台启用状态 enable_twitter: bool = True enable_reddit: bool = True @@ -81,6 +85,8 @@ class SimulationState: "simulation_id": self.simulation_id, "project_id": self.project_id, "graph_id": self.graph_id, + "organization_id": self.organization_id, + "owner_user_id": self.owner_user_id, "enable_twitter": self.enable_twitter, "enable_reddit": self.enable_reddit, "status": self.status.value, @@ -129,12 +135,25 @@ class SimulationManager: '../../uploads/simulations' ) - def __init__(self): - # 确保目录存在 + def __init__( + self, + entity_reader_factory: Optional[Callable[[str], Any]] = None, + session_factory: Optional[Callable[..., Any]] = None, + ): + # Ensure the legacy filesystem cache exists while the durable job migration is in progress. os.makedirs(self.SIMULATION_DATA_DIR, exist_ok=True) - - # 内存中的模拟状态缓存 self._simulations: Dict[str, SimulationState] = {} + self._entity_reader_factory = entity_reader_factory + self._session_factory = session_factory + + def create_entity_reader(self, graph_id: str): + if self._entity_reader_factory is not None: + return self._entity_reader_factory(graph_id) + if Config.MEMORY_BACKEND == "local": + raise ValueError("local_entity_reader_factory_required") + from .zep_entity_reader import ZepEntityReader + + return ZepEntityReader() def _get_simulation_dir(self, simulation_id: str) -> str: """获取模拟数据目录""" @@ -153,6 +172,29 @@ class SimulationManager: json.dump(state.to_dict(), f, ensure_ascii=False, indent=2) self._simulations[state.simulation_id] = state + self._sync_simulation_to_durable(state) + + def _sync_simulation_to_durable(self, state: SimulationState): + """Best-effort mirror of a simulation state into the durable table. + + Only runs when tenant scope is known and a session factory is available; + never raises so the filesystem manager remains authoritative during the + migration. + """ + if not state.organization_id or not callable(self._session_factory): + return + try: + session = self._session_factory() + try: + from .product_repository import ProductRepository + + payload = state.to_dict() + payload["created_by_user_id"] = state.owner_user_id + ProductRepository(session).sync_simulation(payload, commit=True) + finally: + session.close() + except Exception: + logger.debug("durable simulation sync skipped", exc_info=True) def _load_simulation_state(self, simulation_id: str) -> Optional[SimulationState]: """从文件加载模拟状态""" @@ -172,6 +214,8 @@ class SimulationManager: simulation_id=simulation_id, project_id=data.get("project_id", ""), graph_id=data.get("graph_id", ""), + organization_id=data.get("organization_id"), + owner_user_id=data.get("owner_user_id"), enable_twitter=data.get("enable_twitter", True), enable_reddit=data.get("enable_reddit", True), status=SimulationStatus(data.get("status", "created")), @@ -271,18 +315,24 @@ class SimulationManager: # ========== 阶段1: 读取并过滤实体 ========== if progress_callback: - progress_callback("reading", 0, t('progress.connectingZepGraph')) - - reader = ZepEntityReader() + progress_callback("reading", 0, t('progress.connectingGraph' if Config.MEMORY_BACKEND == "local" else 'progress.connectingZepGraph')) + reader = self.create_entity_reader(state.graph_id) + if progress_callback: progress_callback("reading", 30, t('progress.readingNodeData')) - - filtered = reader.filter_defined_entities( - graph_id=state.graph_id, - defined_entity_types=defined_entity_types, - enrich_with_edges=True - ) + + try: + filtered = reader.filter_defined_entities( + graph_id=state.graph_id, + defined_entity_types=defined_entity_types, + enrich_with_edges=True, + ) + except Exception: + close_reader = getattr(reader, "close", None) + if callable(close_reader): + close_reader() + raise state.entities_count = filtered.filtered_count state.entity_types = list(filtered.entity_types) @@ -296,6 +346,9 @@ class SimulationManager: ) if filtered.filtered_count == 0: + close_reader = getattr(reader, "close", None) + if callable(close_reader): + close_reader() state.status = SimulationStatus.FAILED state.error = "没有找到符合条件的实体，请检查图谱是否正确构建" self._save_simulation_state(state) @@ -312,8 +365,18 @@ class SimulationManager: total=total_entities ) - # 传入graph_id以启用Zep检索功能，获取更丰富的上下文 - generator = OasisProfileGenerator(graph_id=state.graph_id) + local_memory_tools = None + if Config.MEMORY_BACKEND == "local": + repository = getattr(reader, "repository", None) + if repository is None: + raise ValueError("local_memory_reader_required") + local_memory_tools = LocalMemoryTools(repository) + + generator = OasisProfileGenerator( + graph_id=state.graph_id, + use_zep_context=Config.MEMORY_BACKEND != "local", + local_memory_tools=local_memory_tools, + ) def profile_progress(current, total, msg): if progress_callback: @@ -336,15 +399,20 @@ class SimulationManager: realtime_output_path = os.path.join(sim_dir, "twitter_profiles.csv") realtime_platform = "twitter" - profiles = generator.generate_profiles_from_entities( - entities=filtered.entities, - use_llm=use_llm_for_profiles, - progress_callback=profile_progress, - graph_id=state.graph_id, # 传入graph_id用于Zep检索 - parallel_count=parallel_profile_count, # 并行生成数量 - realtime_output_path=realtime_output_path, # 实时保存路径 - output_platform=realtime_platform # 输出格式 - ) + try: + profiles = generator.generate_profiles_from_entities( + entities=filtered.entities, + use_llm=use_llm_for_profiles, + progress_callback=profile_progress, + graph_id=state.graph_id, + parallel_count=parallel_profile_count, + realtime_output_path=realtime_output_path, + output_platform=realtime_platform + ) + finally: + close_reader = getattr(reader, "close", None) + if callable(close_reader): + close_reader() state.profiles_count = len(profiles) @@ -448,11 +516,17 @@ class SimulationManager: return state except Exception as e: - logger.error(f"模拟准备失败: {simulation_id}, error={str(e)}") - import traceback - logger.error(traceback.format_exc()) + close_reader = locals().get("reader") + close_method = getattr(close_reader, "close", None) + if callable(close_method): + close_method() + logger.error( + "Simulation preparation failed: simulation_id=%s error_type=%s", + simulation_id, + type(e).__name__, + ) state.status = SimulationStatus.FAILED - state.error = str(e) + state.error = t('api.internalError') self._save_simulation_state(state) raise diff --git a/backend/app/services/simulation_runner.py b/backend/app/services/simulation_runner.py index 620fe0f..9df37f6 100644 --- a/backend/app/services/simulation_runner.py +++ b/backend/app/services/simulation_runner.py @@ -20,8 +20,8 @@ from queue import Queue from ..config import Config from ..utils.logger import get_logger -from ..utils.locale import get_locale, set_locale -from .zep_graph_memory_updater import ZepGraphMemoryManager +from ..utils.locale import get_locale, set_locale, t +from .local_graph_memory_updater import LocalGraphMemoryManager from .simulation_ipc import SimulationIPCClient, CommandType, IPCResponse logger = get_logger('crowdsight.simulation_runner') @@ -308,6 +308,14 @@ class SimulationRunner: json.dump(data, f, ensure_ascii=False, indent=2) cls._run_states[state.simulation_id] = state + + @classmethod + def _graph_memory_manager(cls): + if Config.MEMORY_BACKEND == "local": + return LocalGraphMemoryManager + from .zep_graph_memory_updater import ZepGraphMemoryManager + + return ZepGraphMemoryManager @classmethod def start_simulation( @@ -316,7 +324,9 @@ class SimulationRunner: platform: str = "parallel", # twitter / reddit / parallel max_rounds: int = None, # 最大模拟轮数（可选，用于截断过长的模拟） enable_graph_memory_update: bool = False, # 是否将活动更新到Zep图谱 - graph_id: str = None # Zep图谱ID（启用图谱更新时必需） + graph_id: Optional[str] = None, # 图谱ID（启用图谱更新时必需） + organization_id: Optional[str] = None, + session_factory=None, ) -> SimulationRunState: """ 启动模拟 @@ -375,7 +385,19 @@ class SimulationRunner: raise ValueError("启用图谱记忆更新时必须提供 graph_id") try: - ZepGraphMemoryManager.create_updater(simulation_id, graph_id) + if Config.MEMORY_BACKEND == "local": + if not organization_id or session_factory is None: + raise ValueError("local_graph_memory_scope_required") + LocalGraphMemoryManager.create_updater( + simulation_id, + graph_id, + organization_id=organization_id, + session_factory=session_factory, + ) + else: + from .zep_graph_memory_updater import ZepGraphMemoryManager + + ZepGraphMemoryManager.create_updater(simulation_id, graph_id) cls._graph_memory_enabled[simulation_id] = True logger.info(f"已启用图谱记忆更新: simulation_id={simulation_id}, graph_id={graph_id}") except Exception as e: @@ -471,8 +493,13 @@ class SimulationRunner: logger.info(f"模拟启动成功: {simulation_id}, pid={process.pid}, platform={platform}") except Exception as e: + logger.error( + "Simulation process start failed: simulation_id=%s error_type=%s", + simulation_id, + type(e).__name__, + ) state.runner_status = RunnerStatus.FAILED - state.error = str(e) + state.error = t('api.internalError') cls._save_run_state(state) raise @@ -530,36 +557,38 @@ class SimulationRunner: logger.info(f"模拟完成: {simulation_id}") else: state.runner_status = RunnerStatus.FAILED - # 从主日志文件读取错误信息 - main_log_path = os.path.join(sim_dir, "simulation.log") - error_info = "" - try: - if os.path.exists(main_log_path): - with open(main_log_path, 'r', encoding='utf-8') as f: - error_info = f.read()[-2000:] # 取最后2000字符 - except Exception: - pass - state.error = f"进程退出码: {exit_code}, 错误: {error_info}" - logger.error(f"模拟失败: {simulation_id}, error={state.error}") + state.error = t('api.internalError') + logger.error( + "Simulation process failed: simulation_id=%s exit_code=%s", + simulation_id, + exit_code, + ) state.twitter_running = False state.reddit_running = False cls._save_run_state(state) except Exception as e: - logger.error(f"监控线程异常: {simulation_id}, error={str(e)}") + logger.error( + "Simulation monitor failed: simulation_id=%s error_type=%s", + simulation_id, + type(e).__name__, + ) state.runner_status = RunnerStatus.FAILED - state.error = str(e) + state.error = t('api.internalError') cls._save_run_state(state) finally: # 停止图谱记忆更新器 if cls._graph_memory_enabled.get(simulation_id, False): try: - ZepGraphMemoryManager.stop_updater(simulation_id) + cls._graph_memory_manager().stop_updater(simulation_id) logger.info(f"已停止图谱记忆更新: simulation_id={simulation_id}") except Exception as e: - logger.error(f"停止图谱记忆更新器失败: {e}") + logger.error( + "Stopping graph memory updater failed: error_type=%s", + type(e).__name__, + ) cls._graph_memory_enabled.pop(simulation_id, None) # 清理进程资源 @@ -604,7 +633,7 @@ class SimulationRunner: graph_memory_enabled = cls._graph_memory_enabled.get(state.simulation_id, False) graph_updater = None if graph_memory_enabled: - graph_updater = ZepGraphMemoryManager.get_updater(state.simulation_id) + graph_updater = cls._graph_memory_manager().get_updater(state.simulation_id) try: with open(log_path, 'r', encoding='utf-8') as f: @@ -812,7 +841,7 @@ class SimulationRunner: # 停止图谱记忆更新器 if cls._graph_memory_enabled.get(simulation_id, False): try: - ZepGraphMemoryManager.stop_updater(simulation_id) + cls._graph_memory_manager().stop_updater(simulation_id) logger.info(f"已停止图谱记忆更新: simulation_id={simulation_id}") except Exception as e: logger.error(f"停止图谱记忆更新器失败: {e}") @@ -1206,7 +1235,7 @@ class SimulationRunner: # 首先停止所有图谱记忆更新器（stop_all 内部会打印日志） try: - ZepGraphMemoryManager.stop_all() + cls._graph_memory_manager().stop_all() except Exception as e: logger.error(f"停止图谱记忆更新器失败: {e}") cls._graph_memory_enabled.clear() diff --git a/backend/app/services/usage_service.py b/backend/app/services/usage_service.py new file mode 100644 index 0000000..bbfed1a --- /dev/null +++ b/backend/app/services/usage_service.py @@ -0,0 +1,69 @@ +"""Durable LLM usage/cost accounting service. + +Records per-organization, per-user LLM usage without storing any prompt content +or secrets. A simple default cost estimate (input/output per-token) is applied +and can be overridden by a rate table later. +""" + +from __future__ import annotations + +from typing import Optional + +from sqlalchemy.orm import Session + +from ..models.usage import UsageEvent + +# Default per-1K token cost estimates (USD); a rate table can supersede later. +_DEFAULT_INPUT_RATE_PER_1K = 0.0025 +_DEFAULT_OUTPUT_RATE_PER_1K = 0.0100 + + +class UsageService: + def __init__(self, session: Session): + self.session = session + + def record_event( + self, + *, + organization_id: str, + user_id: Optional[str], + operation: str, + model: Optional[str] = None, + input_tokens: int = 0, + output_tokens: int = 0, + ) -> str: + if not isinstance(organization_id, str) or not organization_id: + raise ValueError("organization_id_required") + cost = ( + (input_tokens / 1000) * _DEFAULT_INPUT_RATE_PER_1K + + (output_tokens / 1000) * _DEFAULT_OUTPUT_RATE_PER_1K + ) + event = UsageEvent( + organization_id=organization_id, + user_id=user_id, + operation=operation, + model=model, + input_tokens=int(input_tokens or 0), + output_tokens=int(output_tokens or 0), + estimated_cost=round(cost, 6), + ) + self.session.add(event) + self.session.flush() + return event.id + + def list_events(self, *, organization_id: str, limit: int = 100) -> list[UsageEvent]: + return ( + self.session.query(UsageEvent) + .filter(UsageEvent.organization_id == organization_id) + .order_by(UsageEvent.created_at.desc()) + .limit(min(max(int(limit), 1), 1000)) + .all() + ) + + def total_cost(self, *, organization_id: str) -> float: + rows = ( + self.session.query(UsageEvent) + .filter(UsageEvent.organization_id == organization_id) + .all() + ) + return round(sum(row.estimated_cost for row in rows), 6) diff --git a/backend/app/services/zep_graph_memory_updater.py b/backend/app/services/zep_graph_memory_updater.py index 97c5488..aa60f20 100644 --- a/backend/app/services/zep_graph_memory_updater.py +++ b/backend/app/services/zep_graph_memory_updater.py @@ -21,182 +21,7 @@ from ..utils.locale import get_locale, set_locale logger = get_logger('crowdsight.zep_graph_memory_updater') -@dataclass -class AgentActivity: - """Agent活动记录""" - platform: str # twitter / reddit - agent_id: int - agent_name: str - action_type: str # CREATE_POST, LIKE_POST, etc. - action_args: Dict[str, Any] - round_num: int - timestamp: str - - def to_episode_text(self) -> str: - """ - 将活动转换为可以发送给Zep的文本描述 - - 采用自然语言描述格式，让Zep能够从中提取实体和关系 - 不添加模拟相关的前缀，避免误导图谱更新 - """ - # 根据不同的动作类型生成不同的描述 - action_descriptions = { - "CREATE_POST": self._describe_create_post, - "LIKE_POST": self._describe_like_post, - "DISLIKE_POST": self._describe_dislike_post, - "REPOST": self._describe_repost, - "QUOTE_POST": self._describe_quote_post, - "FOLLOW": self._describe_follow, - "CREATE_COMMENT": self._describe_create_comment, - "LIKE_COMMENT": self._describe_like_comment, - "DISLIKE_COMMENT": self._describe_dislike_comment, - "SEARCH_POSTS": self._describe_search, - "SEARCH_USER": self._describe_search_user, - "MUTE": self._describe_mute, - } - - describe_func = action_descriptions.get(self.action_type, self._describe_generic) - description = describe_func() - - # 直接返回 "agent名称: 活动描述" 格式，不添加模拟前缀 - return f"{self.agent_name}: {description}" - - def _describe_create_post(self) -> str: - content = self.action_args.get("content", "") - if content: - return f"发布了一条帖子：「{content}」" - return "发布了一条帖子" - - def _describe_like_post(self) -> str: - """点赞帖子 - 包含帖子原文和作者信息""" - post_content = self.action_args.get("post_content", "") - post_author = self.action_args.get("post_author_name", "") - - if post_content and post_author: - return f"点赞了{post_author}的帖子：「{post_content}」" - elif post_content: - return f"点赞了一条帖子：「{post_content}」" - elif post_author: - return f"点赞了{post_author}的一条帖子" - return "点赞了一条帖子" - - def _describe_dislike_post(self) -> str: - """踩帖子 - 包含帖子原文和作者信息""" - post_content = self.action_args.get("post_content", "") - post_author = self.action_args.get("post_author_name", "") - - if post_content and post_author: - return f"踩了{post_author}的帖子：「{post_content}」" - elif post_content: - return f"踩了一条帖子：「{post_content}」" - elif post_author: - return f"踩了{post_author}的一条帖子" - return "踩了一条帖子" - - def _describe_repost(self) -> str: - """转发帖子 - 包含原帖内容和作者信息""" - original_content = self.action_args.get("original_content", "") - original_author = self.action_args.get("original_author_name", "") - - if original_content and original_author: - return f"转发了{original_author}的帖子：「{original_content}」" - elif original_content: - return f"转发了一条帖子：「{original_content}」" - elif original_author: - return f"转发了{original_author}的一条帖子" - return "转发了一条帖子" - - def _describe_quote_post(self) -> str: - """引用帖子 - 包含原帖内容、作者信息和引用评论""" - original_content = self.action_args.get("original_content", "") - original_author = self.action_args.get("original_author_name", "") - quote_content = self.action_args.get("quote_content", "") or self.action_args.get("content", "") - - base = "" - if original_content and original_author: - base = f"引用了{original_author}的帖子「{original_content}」" - elif original_content: - base = f"引用了一条帖子「{original_content}」" - elif original_author: - base = f"引用了{original_author}的一条帖子" - else: - base = "引用了一条帖子" - - if quote_content: - base += f"，并评论道：「{quote_content}」" - return base - - def _describe_follow(self) -> str: - """关注用户 - 包含被关注用户的名称""" - target_user_name = self.action_args.get("target_user_name", "") - - if target_user_name: - return f"关注了用户「{target_user_name}」" - return "关注了一个用户" - - def _describe_create_comment(self) -> str: - """发表评论 - 包含评论内容和所评论的帖子信息""" - content = self.action_args.get("content", "") - post_content = self.action_args.get("post_content", "") - post_author = self.action_args.get("post_author_name", "") - - if content: - if post_content and post_author: - return f"在{post_author}的帖子「{post_content}」下评论道：「{content}」" - elif post_content: - return f"在帖子「{post_content}」下评论道：「{content}」" - elif post_author: - return f"在{post_author}的帖子下评论道：「{content}」" - return f"评论道：「{content}」" - return "发表了评论" - - def _describe_like_comment(self) -> str: - """点赞评论 - 包含评论内容和作者信息""" - comment_content = self.action_args.get("comment_content", "") - comment_author = self.action_args.get("comment_author_name", "") - - if comment_content and comment_author: - return f"点赞了{comment_author}的评论：「{comment_content}」" - elif comment_content: - return f"点赞了一条评论：「{comment_content}」" - elif comment_author: - return f"点赞了{comment_author}的一条评论" - return "点赞了一条评论" - - def _describe_dislike_comment(self) -> str: - """踩评论 - 包含评论内容和作者信息""" - comment_content = self.action_args.get("comment_content", "") - comment_author = self.action_args.get("comment_author_name", "") - - if comment_content and comment_author: - return f"踩了{comment_author}的评论：「{comment_content}」" - elif comment_content: - return f"踩了一条评论：「{comment_content}」" - elif comment_author: - return f"踩了{comment_author}的一条评论" - return "踩了一条评论" - - def _describe_search(self) -> str: - """搜索帖子 - 包含搜索关键词""" - query = self.action_args.get("query", "") or self.action_args.get("keyword", "") - return f"搜索了「{query}」" if query else "进行了搜索" - - def _describe_search_user(self) -> str: - """搜索用户 - 包含搜索关键词""" - query = self.action_args.get("query", "") or self.action_args.get("username", "") - return f"搜索了用户「{query}」" if query else "搜索了用户" - - def _describe_mute(self) -> str: - """屏蔽用户 - 包含被屏蔽用户的名称""" - target_user_name = self.action_args.get("target_user_name", "") - - if target_user_name: - return f"屏蔽了用户「{target_user_name}」" - return "屏蔽了一个用户" - - def _describe_generic(self) -> str: - # 对于未知的动作类型，生成通用描述 - return f"执行了{self.action_type}操作" +from .memory_activity import AgentActivity class ZepGraphMemoryUpdater: diff --git a/backend/app/services/zep_tools.py b/backend/app/services/zep_tools.py index 41e70c0..b41dac5 100644 --- a/backend/app/services/zep_tools.py +++ b/backend/app/services/zep_tools.py @@ -1461,14 +1461,18 @@ Return the sub-questions in JSON format.""" except ValueError as e: # 模拟环境未运行 - logger.warning(t("console.interviewApiCallFailed", error=e)) - result.summary = f"Interview failed: {str(e)}. Simulation environment may have closed. Please ensure OASIS environment is running." + logger.warning( + "Interview API call failed: error_type=%s", + type(e).__name__, + ) + result.summary = t("api.internalError") return result except Exception as e: - logger.error(t("console.interviewApiCallException", error=e)) - import traceback - logger.error(traceback.format_exc()) - result.summary = f"采访过程发生错误：{str(e)}" + logger.error( + "Interview API call exception: error_type=%s", + type(e).__name__, + ) + result.summary = t("api.internalError") return result # Step 6: 生成采访摘要 diff --git a/backend/app/utils/api_errors.py b/backend/app/utils/api_errors.py new file mode 100644 index 0000000..a5c8e6d --- /dev/null +++ b/backend/app/utils/api_errors.py @@ -0,0 +1,43 @@ +"""Structured, localized API error contracts. + +The exception text is intentionally never serialized. Route handlers can raise +``ApiError`` with a stable code and translation key; Flask integration resolves +the message through the current locale at the response boundary. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Callable, Mapping + + +Translator = Callable[..., str] + + +@dataclass +class ApiError(Exception): + """A safe application error that can cross the HTTP boundary.""" + + code: str + status_code: int + message_key: str + params: Mapping[str, object] = field(default_factory=dict) + + def __post_init__(self): + Exception.__init__(self, self.code) + + def to_payload(self, translate: Translator) -> dict[str, object]: + return { + "success": False, + "error_code": self.code, + "message": translate(self.message_key, **dict(self.params)), + } + + +def internal_error_payload(translate: Translator) -> dict[str, object]: + """Return a generic internal error without accepting exception details.""" + return ApiError( + code="internal_error", + status_code=500, + message_key="api.internalError", + ).to_payload(translate) diff --git a/backend/app/utils/language_policy.py b/backend/app/utils/language_policy.py new file mode 100644 index 0000000..73fcf82 --- /dev/null +++ b/backend/app/utils/language_policy.py @@ -0,0 +1,82 @@ +"""Language policy shared by API locale negotiation and background jobs.""" + +from __future__ import annotations + +from typing import Iterable + +SUPPORTED_LOCALES = ("th", "en") +DEFAULT_LOCALE = "th" + +# Legacy values are intentionally mapped to the default locale so an old +# browser preference cannot re-enable an unsupported product language. +_LEGACY_LOCALE_ALIASES = { + "zh": DEFAULT_LOCALE, + "zh-cn": DEFAULT_LOCALE, + "zh-tw": DEFAULT_LOCALE, +} + + +def normalize_locale(value: object, default: str = DEFAULT_LOCALE) -> str: + """Return one supported base locale, failing closed to ``default``.""" + safe_default = default if default in SUPPORTED_LOCALES else DEFAULT_LOCALE + if not isinstance(value, str): + return safe_default + + normalized = value.strip().replace("_", "-").lower() + if not normalized: + return safe_default + if normalized in _LEGACY_LOCALE_ALIASES: + return _LEGACY_LOCALE_ALIASES[normalized] + if normalized in SUPPORTED_LOCALES: + return normalized + + base_locale = normalized.split("-", 1)[0] + if base_locale in SUPPORTED_LOCALES: + return base_locale + if base_locale in _LEGACY_LOCALE_ALIASES: + return _LEGACY_LOCALE_ALIASES[base_locale] + return safe_default + + +def locale_from_accept_language(header: object) -> str: + """Choose the best supported locale from an HTTP Accept-Language value.""" + if not isinstance(header, str) or not header.strip(): + return DEFAULT_LOCALE + + candidates: list[tuple[float, int, str]] = [] + for position, raw_item in enumerate(header.split(",")): + parts = [part.strip() for part in raw_item.split(";")] + language = parts[0] + quality = 1.0 + for parameter in parts[1:]: + key, separator, value = parameter.partition("=") + if key.strip().lower() != "q" or not separator: + continue + try: + quality = float(value.strip()) + except ValueError: + quality = 0.0 + break + + if quality <= 0: + continue + normalized_language = language.strip().replace("_", "-").lower() + base_locale = normalized_language.split("-", 1)[0] + if base_locale in SUPPORTED_LOCALES: + # Higher q wins; original order breaks ties. + candidates.append((quality, -position, base_locale)) + + if not candidates: + return DEFAULT_LOCALE + candidates.sort(reverse=True) + return candidates[0][2] + + +def is_supported_locale(value: object) -> bool: + """Return whether ``value`` is already a canonical supported locale.""" + return isinstance(value, str) and value in SUPPORTED_LOCALES + + +def supported_locales() -> Iterable[str]: + """Expose supported locales without allowing callers to mutate the tuple.""" + return SUPPORTED_LOCALES diff --git a/backend/app/utils/locale.py b/backend/app/utils/locale.py index 23d04aa..440d865 100644 --- a/backend/app/utils/locale.py +++ b/backend/app/utils/locale.py @@ -3,6 +3,13 @@ import os import threading from flask import request, has_request_context +from .language_policy import ( + DEFAULT_LOCALE, + SUPPORTED_LOCALES, + locale_from_accept_language, + normalize_locale, +) + _thread_local = threading.local() _locales_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'locales') @@ -16,25 +23,26 @@ _translations = {} for filename in os.listdir(_locales_dir): if filename.endswith('.json') and filename != 'languages.json': locale_name = filename[:-5] + if locale_name not in SUPPORTED_LOCALES: + continue with open(os.path.join(_locales_dir, filename), 'r', encoding='utf-8') as f: _translations[locale_name] = json.load(f) def set_locale(locale: str): - """Set locale for current thread. Call at the start of background threads.""" - _thread_local.locale = locale + """Set a canonical locale for the current background thread.""" + _thread_local.locale = normalize_locale(locale) def get_locale() -> str: if has_request_context(): - raw = request.headers.get('Accept-Language', 'zh') - return raw if raw in _translations else 'zh' - return getattr(_thread_local, 'locale', 'zh') + return locale_from_accept_language(request.headers.get('Accept-Language', '')) + return normalize_locale(getattr(_thread_local, 'locale', DEFAULT_LOCALE)) def t(key: str, **kwargs) -> str: locale = get_locale() - messages = _translations.get(locale, _translations.get('zh', {})) + messages = _translations.get(locale, _translations.get(DEFAULT_LOCALE, {})) value = messages for part in key.split('.'): @@ -45,7 +53,7 @@ def t(key: str, **kwargs) -> str: break if value is None: - value = _translations.get('zh', {}) + value = _translations.get(DEFAULT_LOCALE, {}) for part in key.split('.'): if isinstance(value, dict): value = value.get(part) @@ -65,5 +73,10 @@ def t(key: str, **kwargs) -> str: def get_language_instruction() -> str: locale = get_locale() - lang_config = _languages.get(locale, _languages.get('zh', {})) - return lang_config.get('llmInstruction', '请使用中文回答。') + lang_config = _languages.get(locale, _languages.get(DEFAULT_LOCALE, {})) + return lang_config.get( + 'llmInstruction', + 'IMPORTANT: Respond exclusively in Thai language.' + if locale == DEFAULT_LOCALE + else 'IMPORTANT: Respond exclusively in English language.', + ) diff --git a/backend/migrations/env.py b/backend/migrations/env.py new file mode 100644 index 0000000..8c21ac9 --- /dev/null +++ b/backend/migrations/env.py @@ -0,0 +1,58 @@ +"""Alembic environment for the SaaS identity schema.""" + +from __future__ import annotations + +import os +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import pool + +from app.db import Base, create_database_engine +from app.models import memory, saas # noqa: F401 - register models on metadata + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + + +target_metadata = Base.metadata + + +def database_url() -> str: + url = os.environ.get("DATABASE_URL") or config.get_main_option("sqlalchemy.url") + if not url: + raise RuntimeError("DATABASE_URL is required for migrations") + return url + + +def run_migrations_offline() -> None: + context.configure( + url=database_url(), + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + compare_type=True, + compare_server_default=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = create_database_engine(database_url(), poolclass=pool.NullPool) + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + compare_server_default=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/backend/migrations/script.py.mako b/backend/migrations/script.py.mako new file mode 100644 index 0000000..0e341ef --- /dev/null +++ b/backend/migrations/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/backend/migrations/versions/0001_identity.py b/backend/migrations/versions/0001_identity.py new file mode 100644 index 0000000..e95f259 --- /dev/null +++ b/backend/migrations/versions/0001_identity.py @@ -0,0 +1,79 @@ +"""Create SaaS identity and tenant membership tables. + +Revision ID: 0001_identity +Revises: +Create Date: 2026-08-23 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "0001_identity" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "organizations", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("name", sa.String(length=160), nullable=False), + sa.Column("slug", sa.String(length=80), nullable=False), + sa.Column("status", sa.String(length=32), server_default=sa.text("'active'"), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_organizations_slug", "organizations", ["slug"], unique=True) + + op.create_table( + "users", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("email_normalized", sa.String(length=320), nullable=False), + sa.Column( + "password_hash", + sa.String(length=512), + server_default=sa.text("'!invite_pending'"), + nullable=False, + ), + sa.Column("status", sa.String(length=32), server_default=sa.text("'active'"), nullable=False), + sa.Column("auth_version", sa.Integer(), server_default=sa.text("0"), nullable=False), + sa.Column("locale", sa.String(length=8), server_default=sa.text("'th'"), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_users_email_normalized", "users", ["email_normalized"], unique=True) + + op.create_table( + "memberships", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("user_id", sa.String(length=64), nullable=False), + sa.Column("organization_id", sa.String(length=64), nullable=False), + sa.Column("role", sa.String(length=11), nullable=False), + sa.Column("status", sa.String(length=32), server_default=sa.text("'active'"), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.CheckConstraint( + "role IN ('super_admin', 'admin', 'user')", + name="ck_membership_role", + ), + sa.ForeignKeyConstraint(["organization_id"], ["organizations.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("user_id", "organization_id", name="uq_membership_user_org"), + ) + op.create_index("ix_memberships_user_id", "memberships", ["user_id"], unique=False) + op.create_index("ix_memberships_organization_id", "memberships", ["organization_id"], unique=False) + + +def downgrade() -> None: + op.drop_index("ix_memberships_organization_id", table_name="memberships") + op.drop_index("ix_memberships_user_id", table_name="memberships") + op.drop_table("memberships") + op.drop_index("ix_users_email_normalized", table_name="users") + op.drop_table("users") + op.drop_index("ix_organizations_slug", table_name="organizations") + op.drop_table("organizations") diff --git a/backend/migrations/versions/0002_sessions.py b/backend/migrations/versions/0002_sessions.py new file mode 100644 index 0000000..8c31e7d --- /dev/null +++ b/backend/migrations/versions/0002_sessions.py @@ -0,0 +1,45 @@ +"""Add revocable opaque authentication sessions. + +Revision ID: 0002_sessions +Revises: 0001_identity +Create Date: 2026-08-23 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "0002_sessions" +down_revision: Union[str, None] = "0001_identity" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "sessions", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("user_id", sa.String(length=64), nullable=False), + sa.Column("membership_id", sa.String(length=64), nullable=False), + sa.Column("token_hash", sa.String(length=64), nullable=False), + sa.Column("auth_version", sa.Integer(), server_default=sa.text("0"), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["membership_id"], ["memberships.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_sessions_user_id", "sessions", ["user_id"], unique=False) + op.create_index("ix_sessions_membership_id", "sessions", ["membership_id"], unique=False) + op.create_index("ix_sessions_token_hash", "sessions", ["token_hash"], unique=True) + + +def downgrade() -> None: + op.drop_index("ix_sessions_token_hash", table_name="sessions") + op.drop_index("ix_sessions_membership_id", table_name="sessions") + op.drop_index("ix_sessions_user_id", table_name="sessions") + op.drop_table("sessions") diff --git a/backend/migrations/versions/0003_memory.py b/backend/migrations/versions/0003_memory.py new file mode 100644 index 0000000..e024ffc --- /dev/null +++ b/backend/migrations/versions/0003_memory.py @@ -0,0 +1,116 @@ +"""Add durable local graph-memory tables.""" + +from alembic import op +import sqlalchemy as sa + + +revision = "0003_memory" +down_revision = "0002_sessions" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "memory_graphs", + sa.Column("id", sa.String(length=128), nullable=False), + sa.Column("organization_id", sa.String(length=64), nullable=False), + sa.Column("project_id", sa.String(length=128), nullable=False), + sa.Column("ontology", sa.JSON(), server_default=sa.text("'{}'"), nullable=False), + sa.Column("status", sa.String(length=32), server_default=sa.text("'ready'"), nullable=False), + sa.Column("version", sa.Integer(), server_default=sa.text("1"), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_memory_graphs_organization_id", "memory_graphs", ["organization_id"], unique=False) + op.create_index("ix_memory_graphs_project_id", "memory_graphs", ["project_id"], unique=False) + + op.create_table( + "memory_episodes", + sa.Column("id", sa.String(length=128), nullable=False), + sa.Column("graph_id", sa.String(length=128), nullable=False), + sa.Column("source_type", sa.String(length=32), nullable=False), + sa.Column("source_ref", sa.String(length=256), nullable=False), + sa.Column("normalized_text", sa.Text(), nullable=False), + sa.Column("summary", sa.Text(), server_default=sa.text("''"), nullable=False), + sa.Column("status", sa.String(length=32), server_default=sa.text("'processed'"), nullable=False), + sa.Column("extractor_version", sa.String(length=64), server_default=sa.text("'v1'"), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.ForeignKeyConstraint(["graph_id"], ["memory_graphs.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("graph_id", "source_type", "source_ref", name="uq_memory_episode_source"), + ) + op.create_index("ix_memory_episodes_graph_id", "memory_episodes", ["graph_id"], unique=False) + op.create_index("ix_memory_episodes_graph_status", "memory_episodes", ["graph_id", "status"], unique=False) + + op.create_table( + "memory_nodes", + sa.Column("id", sa.String(length=128), nullable=False), + sa.Column("graph_id", sa.String(length=128), nullable=False), + sa.Column("canonical_name", sa.String(length=512), nullable=False), + sa.Column("normalized_name", sa.String(length=512), nullable=False), + sa.Column("labels", sa.JSON(), server_default=sa.text("'[]'"), nullable=False), + sa.Column("aliases", sa.JSON(), server_default=sa.text("'[]'"), nullable=False), + sa.Column("attributes", sa.JSON(), server_default=sa.text("'{}'"), nullable=False), + sa.Column("summary", sa.Text(), server_default=sa.text("''"), nullable=False), + sa.Column("confidence", sa.Float(), server_default=sa.text("0"), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.ForeignKeyConstraint(["graph_id"], ["memory_graphs.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("graph_id", "normalized_name", name="uq_memory_node_graph_name"), + ) + op.create_index("ix_memory_nodes_graph_id", "memory_nodes", ["graph_id"], unique=False) + op.create_index("ix_memory_nodes_graph_name", "memory_nodes", ["graph_id", "normalized_name"], unique=False) + + op.create_table( + "memory_edges", + sa.Column("id", sa.String(length=128), nullable=False), + sa.Column("graph_id", sa.String(length=128), nullable=False), + sa.Column("source_node_id", sa.String(length=128), nullable=False), + sa.Column("target_node_id", sa.String(length=128), nullable=False), + sa.Column("relation", sa.String(length=128), nullable=False), + sa.Column("fact", sa.Text(), nullable=False), + sa.Column("attributes", sa.JSON(), server_default=sa.text("'{}'"), nullable=False), + sa.Column("confidence", sa.Float(), server_default=sa.text("0"), nullable=False), + sa.Column("valid_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("invalid_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("expired_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.ForeignKeyConstraint(["graph_id"], ["memory_graphs.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["source_node_id"], ["memory_nodes.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["target_node_id"], ["memory_nodes.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_memory_edges_graph_id", "memory_edges", ["graph_id"], unique=False) + op.create_index("ix_memory_edges_source_node_id", "memory_edges", ["source_node_id"], unique=False) + op.create_index("ix_memory_edges_target_node_id", "memory_edges", ["target_node_id"], unique=False) + op.create_index("ix_memory_edges_graph_relation", "memory_edges", ["graph_id", "relation"], unique=False) + op.create_index( + "ix_memory_edges_graph_temporal", + "memory_edges", + ["graph_id", "valid_at", "invalid_at"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index("ix_memory_edges_graph_temporal", table_name="memory_edges") + op.drop_index("ix_memory_edges_graph_relation", table_name="memory_edges") + op.drop_index("ix_memory_edges_target_node_id", table_name="memory_edges") + op.drop_index("ix_memory_edges_source_node_id", table_name="memory_edges") + op.drop_index("ix_memory_edges_graph_id", table_name="memory_edges") + op.drop_table("memory_edges") + + op.drop_index("ix_memory_nodes_graph_name", table_name="memory_nodes") + op.drop_index("ix_memory_nodes_graph_id", table_name="memory_nodes") + op.drop_table("memory_nodes") + + op.drop_index("ix_memory_episodes_graph_status", table_name="memory_episodes") + op.drop_index("ix_memory_episodes_graph_id", table_name="memory_episodes") + op.drop_table("memory_episodes") + + op.drop_index("ix_memory_graphs_project_id", table_name="memory_graphs") + op.drop_index("ix_memory_graphs_organization_id", table_name="memory_graphs") + op.drop_table("memory_graphs") diff --git a/backend/migrations/versions/0004_operations.py b/backend/migrations/versions/0004_operations.py new file mode 100644 index 0000000..931a1ca --- /dev/null +++ b/backend/migrations/versions/0004_operations.py @@ -0,0 +1,107 @@ +"""Add durable jobs, idempotency records, and audit logs.""" + +from alembic import op +import sqlalchemy as sa + + +revision = "0004_operations" +down_revision = "0003_memory" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "jobs", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("organization_id", sa.String(length=64), nullable=False), + sa.Column("owner_user_id", sa.String(length=64), nullable=True), + sa.Column("project_id", sa.String(length=128), nullable=True), + sa.Column("graph_id", sa.String(length=128), nullable=True), + sa.Column("operation", sa.String(length=120), nullable=False), + sa.Column("status", sa.String(length=32), server_default=sa.text("'queued'"), nullable=False), + sa.Column("progress", sa.Integer(), server_default=sa.text("0"), nullable=False), + sa.Column("error_code", sa.String(length=120), nullable=True), + sa.Column("result_ref", sa.String(length=512), nullable=True), + sa.Column("idempotency_key", sa.String(length=128), nullable=True), + sa.Column("attempt", sa.Integer(), server_default=sa.text("0"), nullable=False), + sa.Column("settings_version", sa.String(length=128), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.CheckConstraint( + "status IN ('queued', 'running', 'succeeded', 'failed', 'cancelled')", + name="ck_jobs_status", + ), + sa.ForeignKeyConstraint(["organization_id"], ["organizations.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["owner_user_id"], ["users.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_jobs_organization_id", "jobs", ["organization_id"], unique=False) + op.create_index("ix_jobs_owner_user_id", "jobs", ["owner_user_id"], unique=False) + op.create_index("ix_jobs_project_id", "jobs", ["project_id"], unique=False) + op.create_index("ix_jobs_graph_id", "jobs", ["graph_id"], unique=False) + op.create_index("ix_jobs_org_status_created", "jobs", ["organization_id", "status", "created_at"], unique=False) + op.create_index("ix_jobs_org_owner", "jobs", ["organization_id", "owner_user_id"], unique=False) + + op.create_table( + "idempotency_records", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("organization_id", sa.String(length=64), nullable=False), + sa.Column("user_id", sa.String(length=64), nullable=False), + sa.Column("key", sa.String(length=128), nullable=False), + sa.Column("request_hash", sa.String(length=64), nullable=False), + sa.Column("status", sa.String(length=32), server_default=sa.text("'reserved'"), nullable=False), + sa.Column("response_status", sa.Integer(), nullable=True), + sa.Column("response_body", sa.JSON(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["organization_id"], ["organizations.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("organization_id", "user_id", "key", name="uq_idempotency_org_user_key"), + ) + op.create_index("ix_idempotency_records_organization_id", "idempotency_records", ["organization_id"], unique=False) + op.create_index("ix_idempotency_records_user_id", "idempotency_records", ["user_id"], unique=False) + op.create_index("ix_idempotency_expiry", "idempotency_records", ["expires_at"], unique=False) + + op.create_table( + "audit_logs", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("organization_id", sa.String(length=64), nullable=False), + sa.Column("actor_user_id", sa.String(length=64), nullable=True), + sa.Column("action", sa.String(length=160), nullable=False), + sa.Column("target_type", sa.String(length=80), nullable=False), + sa.Column("target_id", sa.String(length=160), nullable=True), + sa.Column("metadata", sa.JSON(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.ForeignKeyConstraint(["organization_id"], ["organizations.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["actor_user_id"], ["users.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_audit_logs_organization_id", "audit_logs", ["organization_id"], unique=False) + op.create_index("ix_audit_logs_actor_user_id", "audit_logs", ["actor_user_id"], unique=False) + op.create_index("ix_audit_org_created", "audit_logs", ["organization_id", "created_at"], unique=False) + op.create_index("ix_audit_org_target", "audit_logs", ["organization_id", "target_type", "target_id"], unique=False) + + +def downgrade() -> None: + op.drop_index("ix_audit_org_target", table_name="audit_logs") + op.drop_index("ix_audit_org_created", table_name="audit_logs") + op.drop_index("ix_audit_logs_actor_user_id", table_name="audit_logs") + op.drop_index("ix_audit_logs_organization_id", table_name="audit_logs") + op.drop_table("audit_logs") + + op.drop_index("ix_idempotency_expiry", table_name="idempotency_records") + op.drop_index("ix_idempotency_records_user_id", table_name="idempotency_records") + op.drop_index("ix_idempotency_records_organization_id", table_name="idempotency_records") + op.drop_table("idempotency_records") + + op.drop_index("ix_jobs_org_owner", table_name="jobs") + op.drop_index("ix_jobs_org_status_created", table_name="jobs") + op.drop_index("ix_jobs_graph_id", table_name="jobs") + op.drop_index("ix_jobs_project_id", table_name="jobs") + op.drop_index("ix_jobs_owner_user_id", table_name="jobs") + op.drop_index("ix_jobs_organization_id", table_name="jobs") + op.drop_table("jobs") diff --git a/backend/migrations/versions/0005_job_payload.py b/backend/migrations/versions/0005_job_payload.py new file mode 100644 index 0000000..3fd713a --- /dev/null +++ b/backend/migrations/versions/0005_job_payload.py @@ -0,0 +1,22 @@ +"""Persist task message, result, and progress detail in durable jobs.""" + +from alembic import op +import sqlalchemy as sa + + +revision = "0005_job_payload" +down_revision = "0004_operations" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("jobs", sa.Column("message", sa.Text(), server_default=sa.text("''"), nullable=False)) + op.add_column("jobs", sa.Column("result", sa.JSON(), nullable=True)) + op.add_column("jobs", sa.Column("progress_detail", sa.JSON(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("jobs", "progress_detail") + op.drop_column("jobs", "result") + op.drop_column("jobs", "message") diff --git a/backend/migrations/versions/0006_job_metadata.py b/backend/migrations/versions/0006_job_metadata.py new file mode 100644 index 0000000..1670c16 --- /dev/null +++ b/backend/migrations/versions/0006_job_metadata.py @@ -0,0 +1,18 @@ +"""Add durable job metadata payload.""" + +from alembic import op +import sqlalchemy as sa + + +revision = "0006_job_metadata" +down_revision = "0005_job_payload" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("jobs", sa.Column("metadata", sa.JSON(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("jobs", "metadata") diff --git a/backend/migrations/versions/0007_product_resources.py b/backend/migrations/versions/0007_product_resources.py new file mode 100644 index 0000000..6ff5f3d --- /dev/null +++ b/backend/migrations/versions/0007_product_resources.py @@ -0,0 +1,119 @@ +"""Add durable product-resource tables: projects, simulations, and reports. + +These replace the legacy filesystem-backed project/simulation/report payloads +with tenant- and owner-scoped SQL rows. +""" + +from alembic import op +import sqlalchemy as sa + +revision = "0007_product_resources" +down_revision = "0006_job_metadata" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "projects", + sa.Column("id", sa.String(length=128), nullable=False), + sa.Column("organization_id", sa.String(length=64), nullable=False), + sa.Column("owner_user_id", sa.String(length=64), nullable=True), + sa.Column("name", sa.String(length=255), server_default=sa.text("''"), nullable=False), + sa.Column("status", sa.String(length=32), server_default=sa.text("'created'"), nullable=False), + sa.Column("language", sa.String(length=16), server_default=sa.text("'en'"), nullable=False), + sa.Column("total_text_length", sa.Integer(), server_default=sa.text("0"), nullable=False), + sa.Column("source_metadata", sa.JSON(), nullable=True), + sa.Column("ontology", sa.JSON(), nullable=True), + sa.Column("analysis_summary", sa.Text(), nullable=True), + sa.Column("simulation_requirement", sa.Text(), nullable=True), + sa.Column("graph_id", sa.String(length=128), nullable=True), + sa.Column("graph_build_task_id", sa.String(length=128), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.ForeignKeyConstraint(["organization_id"], ["organizations.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["owner_user_id"], ["users.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_projects_organization_id", "projects", ["organization_id"]) + op.create_index("ix_projects_owner_user_id", "projects", ["owner_user_id"]) + op.create_index("ix_projects_graph_id", "projects", ["graph_id"]) + op.create_index("ix_projects_org_owner", "projects", ["organization_id", "owner_user_id"]) + op.create_index("ix_projects_org_created", "projects", ["organization_id", "created_at"]) + + op.create_table( + "simulations", + sa.Column("id", sa.String(length=128), nullable=False), + sa.Column("organization_id", sa.String(length=64), nullable=False), + sa.Column("project_id", sa.String(length=128), nullable=False), + sa.Column("created_by_user_id", sa.String(length=64), nullable=True), + sa.Column("status", sa.String(length=32), server_default=sa.text("'created'"), nullable=False), + sa.Column("platform", sa.String(length=32), server_default=sa.text("'parallel'"), nullable=False), + sa.Column("config", sa.JSON(), nullable=True), + sa.Column("current_round", sa.Integer(), server_default=sa.text("0"), nullable=False), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["organization_id"], ["organizations.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_simulations_organization_id", "simulations", ["organization_id"]) + op.create_index("ix_simulations_project_id", "simulations", ["project_id"]) + op.create_index("ix_simulations_org_project", "simulations", ["organization_id", "project_id"]) + op.create_index("ix_simulations_org_created", "simulations", ["organization_id", "created_at"]) + + op.create_table( + "reports", + sa.Column("id", sa.String(length=128), nullable=False), + sa.Column("organization_id", sa.String(length=64), nullable=False), + sa.Column("project_id", sa.String(length=128), nullable=False), + sa.Column("simulation_id", sa.String(length=128), nullable=True), + sa.Column("created_by_user_id", sa.String(length=64), nullable=True), + sa.Column("status", sa.String(length=32), server_default=sa.text("'draft'"), nullable=False), + sa.Column("title", sa.String(length=255), server_default=sa.text("''"), nullable=False), + sa.Column("outline", sa.JSON(), nullable=True), + sa.Column("markdown_content", sa.Text(), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["organization_id"], ["organizations.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["simulation_id"], ["simulations.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("organization_id", "id", name="uq_reports_org_id"), + ) + op.create_index("ix_reports_organization_id", "reports", ["organization_id"]) + op.create_index("ix_reports_project_id", "reports", ["project_id"]) + op.create_index("ix_reports_simulation_id", "reports", ["simulation_id"]) + op.create_index("ix_reports_org_project", "reports", ["organization_id", "project_id"]) + op.create_index("ix_reports_org_simulation", "reports", ["organization_id", "simulation_id"]) + op.create_index("ix_reports_org_created", "reports", ["organization_id", "created_at"]) + + +def downgrade() -> None: + op.drop_index("ix_reports_org_created", table_name="reports") + op.drop_index("ix_reports_org_simulation", table_name="reports") + op.drop_index("ix_reports_org_project", table_name="reports") + op.drop_index("ix_reports_simulation_id", table_name="reports") + op.drop_index("ix_reports_project_id", table_name="reports") + op.drop_index("ix_reports_organization_id", table_name="reports") + op.drop_table("reports") + + op.drop_index("ix_simulations_org_created", table_name="simulations") + op.drop_index("ix_simulations_org_project", table_name="simulations") + op.drop_index("ix_simulations_project_id", table_name="simulations") + op.drop_index("ix_simulations_organization_id", table_name="simulations") + op.drop_table("simulations") + + op.drop_index("ix_projects_org_created", table_name="projects") + op.drop_index("ix_projects_org_owner", table_name="projects") + op.drop_index("ix_projects_graph_id", table_name="projects") + op.drop_index("ix_projects_owner_user_id", table_name="projects") + op.drop_index("ix_projects_organization_id", table_name="projects") + op.drop_table("projects") diff --git a/backend/migrations/versions/0008_platform_settings.py b/backend/migrations/versions/0008_platform_settings.py new file mode 100644 index 0000000..046ee5f --- /dev/null +++ b/backend/migrations/versions/0008_platform_settings.py @@ -0,0 +1,28 @@ +"""Add durable, versioned platform settings with redacted secrets.""" + +from alembic import op +import sqlalchemy as sa + +revision = "0008_platform_settings" +down_revision = "0007_product_resources" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "platform_settings", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("version", sa.String(length=64), nullable=False), + sa.Column("settings", sa.JSON(), nullable=True), + sa.Column("secret_ref", sa.String(length=512), nullable=True), + sa.Column("updated_by_user_id", sa.String(length=64), nullable=True), + sa.Column("active", sa.Boolean(), server_default=sa.text("0"), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("version", name="uq_platform_settings_version"), + ) + + +def downgrade() -> None: + op.drop_table("platform_settings") diff --git a/backend/migrations/versions/0009_rate_limit.py b/backend/migrations/versions/0009_rate_limit.py new file mode 100644 index 0000000..9ae67ef --- /dev/null +++ b/backend/migrations/versions/0009_rate_limit.py @@ -0,0 +1,41 @@ +"""Add durable rate-limit event records.""" + +from alembic import op +import sqlalchemy as sa + +revision = "0009_rate_limit" +down_revision = "0008_platform_settings" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "rate_limit_events", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("operation", sa.String(length=120), nullable=False), + sa.Column("key", sa.String(length=255), nullable=False), + sa.Column("organization_id", sa.String(length=64), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_rate_limit_events_key", "rate_limit_events", ["key"], unique=False) + op.create_index( + "ix_rate_limit_op_key_created", + "rate_limit_events", + ["operation", "key", "created_at"], + unique=False, + ) + op.create_index( + "ix_rate_limit_org_created", + "rate_limit_events", + ["organization_id", "created_at"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index("ix_rate_limit_org_created", table_name="rate_limit_events") + op.drop_index("ix_rate_limit_op_key_created", table_name="rate_limit_events") + op.drop_index("ix_rate_limit_events_key", table_name="rate_limit_events") + op.drop_table("rate_limit_events") diff --git a/backend/migrations/versions/0010_usage_events.py b/backend/migrations/versions/0010_usage_events.py new file mode 100644 index 0000000..21b644a --- /dev/null +++ b/backend/migrations/versions/0010_usage_events.py @@ -0,0 +1,39 @@ +"""Add durable LLM usage/cost events.""" + +from alembic import op +import sqlalchemy as sa + +revision = "0010_usage_events" +down_revision = "0009_rate_limit" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "usage_events", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("organization_id", sa.String(length=64), nullable=False), + sa.Column("user_id", sa.String(length=64), nullable=True), + sa.Column("operation", sa.String(length=160), nullable=False), + sa.Column("model", sa.String(length=120), nullable=True), + sa.Column("input_tokens", sa.Integer(), server_default=sa.text("0"), nullable=False), + sa.Column("output_tokens", sa.Integer(), server_default=sa.text("0"), nullable=False), + sa.Column("estimated_cost", sa.Float(), server_default=sa.text("0"), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.ForeignKeyConstraint(["organization_id"], ["organizations.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_usage_events_organization_id", "usage_events", ["organization_id"], unique=False) + op.create_index("ix_usage_events_user_id", "usage_events", ["user_id"], unique=False) + op.create_index("ix_usage_org_created", "usage_events", ["organization_id", "created_at"], unique=False) + op.create_index("ix_usage_org_user", "usage_events", ["organization_id", "user_id"], unique=False) + + +def downgrade() -> None: + op.drop_index("ix_usage_org_user", table_name="usage_events") + op.drop_index("ix_usage_org_created", table_name="usage_events") + op.drop_index("ix_usage_events_user_id", table_name="usage_events") + op.drop_index("ix_usage_events_organization_id", table_name="usage_events") + op.drop_table("usage_events") diff --git a/backend/migrations/versions/0011_password_reset_tokens.py b/backend/migrations/versions/0011_password_reset_tokens.py new file mode 100644 index 0000000..edb30ce --- /dev/null +++ b/backend/migrations/versions/0011_password_reset_tokens.py @@ -0,0 +1,30 @@ +"""Add durable, single-use password reset tokens.""" + +from alembic import op +import sqlalchemy as sa + +revision = "0011_password_reset_tokens" +down_revision = "0010_usage_events" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "password_reset_tokens", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("user_id", sa.String(length=64), nullable=False), + sa.Column("token_hash", sa.String(length=128), nullable=False), + sa.Column("auth_version", sa.Integer(), server_default=sa.text("0"), nullable=False), + sa.Column("used", sa.Boolean(), server_default=sa.text("0"), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_password_reset_tokens_user_id", "password_reset_tokens", ["user_id"], unique=False) + + +def downgrade() -> None: + op.drop_index("ix_password_reset_tokens_user_id", table_name="password_reset_tokens") + op.drop_table("password_reset_tokens") diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 16e337e..835bfa6 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -12,26 +12,27 @@ dependencies = [ # 核心框架 "flask>=3.0.0", "flask-cors>=6.0.0", - # LLM 相关 "openai>=1.0.0", - # Zep Cloud "zep-cloud==3.13.0", - # OASIS 社交媒体模拟 "camel-oasis==0.2.5", "camel-ai==0.2.78", - # 文件处理 "PyMuPDF>=1.24.0", # 编码检测（支持非UTF-8编码的文本文件） "charset-normalizer>=3.0.0", "chardet>=5.0.0", - # 工具库 "python-dotenv>=1.0.0", "pydantic>=2.0.0", + "sqlalchemy>=2.0,<3", + "gunicorn>=21.2.0", + "alembic>=1.13,<2", + "argon2-cffi>=23.1", + "psycopg[binary]>=3.2", + "cryptography>=41.0", ] [project.optional-dependencies] diff --git a/backend/tests/fixtures/memory_parity/entity_reader_fixture.json b/backend/tests/fixtures/memory_parity/entity_reader_fixture.json new file mode 100644 index 0000000..edf7917 --- /dev/null +++ b/backend/tests/fixtures/memory_parity/entity_reader_fixture.json @@ -0,0 +1,49 @@ +{ + "graph_id": "graph-fixture", + "nodes": [ + { + "id": "node-alice", + "canonical_name": "Alice", + "normalized_name": "alice", + "labels": ["Person"], + "aliases": [], + "attributes": {"role": "founder"}, + "summary": "A founder." + }, + { + "id": "node-acme", + "canonical_name": "Acme", + "normalized_name": "acme", + "labels": ["Organization"], + "aliases": [], + "attributes": {}, + "summary": "A company." + }, + { + "id": "node-note", + "canonical_name": "Launch note", + "normalized_name": "launch note", + "labels": ["Note"], + "aliases": [], + "attributes": {}, + "summary": "A source note." + } + ], + "edges": [ + { + "id": "edge-alice-acme", + "source_node_id": "node-alice", + "target_node_id": "node-acme", + "relation": "WORKS_FOR", + "fact": "Alice works for Acme.", + "attributes": {}, + "confidence": 0.9 + } + ], + "filter": { + "defined_entity_types": ["Person", "Organization"], + "total_count": 3, + "filtered_count": 2, + "entity_types": ["Organization", "Person"] + } +} diff --git a/backend/tests/fixtures/memory_parity/tools_fixture.json b/backend/tests/fixtures/memory_parity/tools_fixture.json new file mode 100644 index 0000000..730d6eb --- /dev/null +++ b/backend/tests/fixtures/memory_parity/tools_fixture.json @@ -0,0 +1,75 @@ +{ + "graph_id": "graph-tools-fixture", + "organization_id": "org-a", + "project_id": "project-tools", + "nodes": [ + { + "id": "node-alice", + "canonical_name": "Alice", + "normalized_name": "alice", + "labels": ["Entity", "Person"], + "aliases": ["A. Example"], + "attributes": {"role": "founder"}, + "summary": "A founder mentioned in the source.", + "confidence": 0.91 + }, + { + "id": "node-acme", + "canonical_name": "Acme", + "normalized_name": "acme", + "labels": ["Entity", "Organization"], + "aliases": [], + "attributes": {"sector": "software"}, + "summary": "A software company mentioned in the source.", + "confidence": 0.88 + }, + { + "id": "node-beta", + "canonical_name": "Beta", + "normalized_name": "beta", + "labels": ["Entity", "Organization"], + "aliases": [], + "attributes": {}, + "summary": "A partner company mentioned in the source.", + "confidence": 0.82 + } + ], + "edges": [ + { + "id": "edge-current", + "source_node_id": "node-alice", + "target_node_id": "node-acme", + "relation": "WORKS_FOR", + "fact": "Alice works for Acme.", + "attributes": {}, + "confidence": 0.9, + "valid_at": "2024-01-01T00:00:00+00:00", + "invalid_at": null, + "expired_at": null + }, + { + "id": "edge-historical", + "source_node_id": "node-alice", + "target_node_id": "node-acme", + "relation": "WORKED_FOR", + "fact": "Alice previously worked for Acme.", + "attributes": {}, + "confidence": 0.86, + "valid_at": "2020-01-01T00:00:00+00:00", + "invalid_at": "2023-01-01T00:00:00+00:00", + "expired_at": null + }, + { + "id": "edge-unrelated", + "source_node_id": "node-acme", + "target_node_id": "node-beta", + "relation": "PARTNERED_WITH", + "fact": "Acme partnered with Beta.", + "attributes": {}, + "confidence": 0.8, + "valid_at": "2022-01-01T00:00:00+00:00", + "invalid_at": null, + "expired_at": null + } + ] +} diff --git a/backend/tests/test_admin_users_api.py b/backend/tests/test_admin_users_api.py new file mode 100644 index 0000000..29507d3 --- /dev/null +++ b/backend/tests/test_admin_users_api.py @@ -0,0 +1,235 @@ +import json + +from flask import Flask +from sqlalchemy import create_engine + +from app.api.admin import admin_bp +from app.api.auth import auth_bp +from app.db import Base, create_session_factory +from app.services.identity import IdentityRepository, PasswordService + + +def make_admin_app(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + + app = Flask(__name__) + app.config.update(TESTING=True, SECRET_KEY="test-secret", SESSION_COOKIE_SECURE=False) + app.extensions["crowdsight_session_factory"] = session_factory + app.register_blueprint(auth_bp, url_prefix="/api/auth") + app.register_blueprint(admin_bp, url_prefix="/api/admin") + + with session_factory() as session: + repo = IdentityRepository(session) + org = repo.create_organization(name="Org A", slug="org-a") + admin = repo.create_user( + email="admin@example.com", + password_hash=PasswordService.hash_password("correct horse battery staple"), + ) + repo.create_membership(admin.id, org.id, "admin") + user = repo.create_user( + email="user@example.com", + password_hash=PasswordService.hash_password("correct horse battery staple"), + ) + repo.create_membership(user.id, org.id, "user") + session.commit() + + return app, engine + + +def login(client, email): + response = client.post( + "/api/auth/login", + json={"email": email, "password": "correct horse battery staple"}, + ) + assert response.status_code == 200 + + +def _csrf_headers(client): + return {"X-CSRF-Token": client.get_cookie("crowdsight_csrf").value} + + +def test_admin_lists_and_creates_users_only_in_current_organization(): + app, engine = make_admin_app() + try: + client = app.test_client() + login(client, "admin@example.com") + + listed = client.get("/api/admin/users") + assert listed.status_code == 200 + body = listed.get_json() + assert body["data"]["count"] == 2 + assert all("password_hash" not in user for user in body["data"]["users"]) + + created = client.post( + "/api/admin/users", + json={"email": "new-user@example.com"}, + headers=_csrf_headers(client), + ) + assert created.status_code == 201 + assert created.get_json()["data"]["role"] == "user" + finally: + engine.dispose() + + +def test_admin_cannot_grant_admin_and_user_cannot_manage_users(): + app, engine = make_admin_app() + try: + admin_client = app.test_client() + login(admin_client, "admin@example.com") + forbidden = admin_client.post( + "/api/admin/users", + json={"email": "new-admin@example.com", "role": "admin"}, + headers=_csrf_headers(admin_client), + ) + assert forbidden.status_code == 403 + assert forbidden.get_json()["error_code"] == "admin_role_grant_forbidden" + + user_client = app.test_client() + login(user_client, "user@example.com") + user_forbidden = user_client.get("/api/admin/users") + assert user_forbidden.status_code == 403 + finally: + engine.dispose() + +def test_admin_duplicate_email_returns_conflict_without_db_error(): + app, engine = make_admin_app() + try: + client = app.test_client() + login(client, "admin@example.com") + response = client.post( + "/api/admin/users", + json={"email": "user@example.com"}, + headers=_csrf_headers(client), + ) + assert response.status_code == 409 + assert response.get_json()["error_code"] == "user_exists" + finally: + engine.dispose() + +def test_state_changing_admin_request_requires_csrf_token(): + app, engine = make_admin_app() + try: + client = app.test_client() + login(client, "admin@example.com") + response = client.post( + "/api/admin/users", + json={"email": "csrf-user@example.com"}, + ) + assert response.status_code == 403 + assert response.get_json()["error_code"] == "csrf_failed" + finally: + engine.dispose() + + +def test_admin_can_update_user_role_within_allowed_domain(): + app, engine = make_admin_app() + try: + client = app.test_client() + login(client, "admin@example.com") + + # Find the target user's id. + listed = client.get("/api/admin/users").get_json() + target = next(u for u in listed["data"]["users"] if u["email"] == "user@example.com") + + # Admin may manage a USER role only (policy: user→user is allowed). + response = client.patch( + f"/api/admin/users/{target['id']}", + json={"role": "user"}, + headers=_csrf_headers(client), + ) + assert response.status_code == 200 + assert response.get_json()["data"]["role"] == "user" + finally: + engine.dispose() + + +def test_admin_cannot_promote_to_super_admin(): + app, engine = make_admin_app() + try: + client = app.test_client() + login(client, "admin@example.com") + listed = client.get("/api/admin/users").get_json() + target = next(u for u in listed["data"]["users"] if u["email"] == "user@example.com") + + response = client.patch( + f"/api/admin/users/{target['id']}", + json={"role": "super_admin"}, + headers=_csrf_headers(client), + ) + assert response.status_code == 403 + assert response.get_json()["error_code"] == "admin_role_grant_forbidden" + finally: + engine.dispose() + + +def test_settings_endpoint_requires_super_admin(): + app, engine = make_admin_app() + try: + client = app.test_client() + login(client, "admin@example.com") + # An org admin (ADMIN) must NOT be able to read or mutate platform settings. + forbidden_get = client.get("/api/admin/settings") + assert forbidden_get.status_code == 403 + forbidden_put = client.put( + "/api/admin/settings", + json={"provider": "openai", "model": "gpt-4o"}, + headers=_csrf_headers(client), + ) + assert forbidden_put.status_code == 403 + finally: + engine.dispose() + + +def test_super_admin_can_save_and_read_settings(): + app, engine = make_admin_app() + try: + # SettingsService derives its Fernet key from Config.SECRET_KEY; set it + # directly (it was evaluated from env at import time). + from app.config import Config + + Config.SECRET_KEY = "test-encryption-secret-" * 3 + + # Promote the existing admin membership to super_admin in its org. + with app.extensions["crowdsight_session_factory"]() as session: + from sqlalchemy import select + + from app.models.saas import Membership, Organization + from app.services.identity import IdentityRepository + + repo = IdentityRepository(session) + admin_user = repo.get_user_by_email("admin@example.com") + org = session.execute( + select(Organization).where(Organization.slug == "org-a") + ).scalar_one() + membership = session.execute( + select(Membership).where( + Membership.user_id == admin_user.id, + Membership.organization_id == org.id, + ) + ).scalar_one() + membership.role = "super_admin" + session.commit() + + client = app.test_client() + login(client, "admin@example.com") + + saved = client.put( + "/api/admin/settings", + json={"provider": "openai", "model": "gpt-4o", "api_key": "sk-secret-value"}, + headers=_csrf_headers(client), + ) + assert saved.status_code == 200 + data = saved.get_json()["data"] + assert data["settings"]["model"] == "gpt-4o" + # API key is masked (never plaintext). It may be a bare mask or a + # masked-with-prefix variant; the important invariant is no plaintext. + assert "sk-secret-value" not in json.dumps(saved.get_json()) + + # Reading back returns masked value only; no plaintext secret leaked. + got = client.get("/api/admin/settings") + assert got.status_code == 200 + assert "sk-secret-value" not in json.dumps(got.get_json()) + finally: + engine.dispose() diff --git a/backend/tests/test_api_errors.py b/backend/tests/test_api_errors.py new file mode 100644 index 0000000..a8474a2 --- /dev/null +++ b/backend/tests/test_api_errors.py @@ -0,0 +1,48 @@ +import importlib.util +from pathlib import Path +import sys +import unittest + + +_MODULE_PATH = Path(__file__).parents[1] / "app" / "utils" / "api_errors.py" +_SPEC = importlib.util.spec_from_file_location("api_errors_under_test", _MODULE_PATH) +assert _SPEC is not None and _SPEC.loader is not None +_MODULE = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _MODULE +_SPEC.loader.exec_module(_MODULE) + +ApiError = _MODULE.ApiError +internal_error_payload = _MODULE.internal_error_payload + + +class ApiErrorTests(unittest.TestCase): + def translator(self, key, **kwargs): + return f"{key}:{kwargs.get('id', '')}".rstrip(":") + + def test_structured_error_has_code_and_localized_message_only(self): + error = ApiError( + code="project_not_found", + status_code=404, + message_key="api.projectNotFound", + params={"id": "demo"}, + ) + payload = error.to_payload(self.translator) + self.assertEqual(payload, { + "success": False, + "error_code": "project_not_found", + "message": "api.projectNotFound:demo", + }) + self.assertNotIn("error", payload) + self.assertNotIn("details", payload) + + def test_internal_error_never_contains_exception_text(self): + payload = internal_error_payload(self.translator) + self.assertEqual(payload["success"], False) + self.assertEqual(payload["error_code"], "internal_error") + self.assertEqual(payload["message"], "api.internalError") + self.assertNotIn("Traceback", str(payload)) + self.assertNotIn("secret", str(payload)) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/tests/test_api_no_raw_exception_details.py b/backend/tests/test_api_no_raw_exception_details.py new file mode 100644 index 0000000..8f425c9 --- /dev/null +++ b/backend/tests/test_api_no_raw_exception_details.py @@ -0,0 +1,36 @@ +from pathlib import Path +import re + + +API_DIR = Path(__file__).parents[1] / "app" / "api" + + +UNSAFE_PATTERNS = ( + re.compile(r"[\"']traceback[\"']\s*:\s*traceback\.format_exc\(\)"), + re.compile(r"[\"']error[\"']\s*:\s*str\((?:e|exc)\)"), + re.compile(r"t\([^\n]*error\s*=\s*str\((?:e|exc)\)"), + re.compile(r"(?:project|state)\.error\s*=\s*str\((?:e|exc)\)"), + re.compile(r"fail_task\([^\n]*str\((?:e|exc)\)"), +) + + +TARGET_FILES = ( + "simulation.py", + "graph.py", + "report.py", + "agent_group.py", + "template.py", +) + + +def test_api_does_not_expose_or_persist_raw_exception_details(): + findings = [] + for filename in TARGET_FILES: + path = API_DIR / filename + source = path.read_text(encoding="utf-8") + for pattern in UNSAFE_PATTERNS: + for match in pattern.finditer(source): + line = source.count("\n", 0, match.start()) + 1 + findings.append(f"{filename}:{line}:{match.group(0)}") + + assert findings == [], "unsafe exception details found:\n" + "\n".join(findings) diff --git a/backend/tests/test_artifact_store.py b/backend/tests/test_artifact_store.py new file mode 100644 index 0000000..64b77c1 --- /dev/null +++ b/backend/tests/test_artifact_store.py @@ -0,0 +1,70 @@ +"""TDD gate: tenant-scoped artifact store abstraction. + +The legacy project/simulation/report managers write to raw os.path.join under a +shared upload root. This gate proves a scoped ArtifactStore resolves paths +within the tenant's own directory, rejects traversal/absolute components, and +exposes a store/read/delete interface that can later be backed by object +storage without changing callers. +""" + +import pytest + +from app.services.artifact_store import ArtifactStore + + +def test_artifact_store_scopes_path_to_root(monkeypatch, tmp_path): + store = ArtifactStore(str(tmp_path)) + project_dir = store.path_for("org-a", "project-1") + assert str(tmp_path) in project_dir + assert "org-a" in project_dir + assert "project-1" in project_dir + # Tenant-scoped directory lands under the configured root. + assert project_dir.startswith(str(tmp_path)) + + +def test_artifact_store_isolates_tenants(monkeypatch, tmp_path): + store = ArtifactStore(str(tmp_path)) + path_a = store.path_for("org-a", "proj-x", "state.json") + path_b = store.path_for("org-b", "proj-x", "state.json") + assert path_a != path_b + assert "/org-a/" in path_a.replace("\\", "/") + assert "/org-b/" in path_b.replace("\\", "/") + + +@pytest.mark.parametrize( + "bad_segment", + ["../evil", "a/../../etc", "/absolute", "..", ".../..", "a//..", "\\evil", "a/..\\b"], +) +def test_artifact_store_rejects_traversal(bad_segment, tmp_path): + store = ArtifactStore(str(tmp_path)) + with pytest.raises(ValueError): + store.path_for("org-a", bad_segment) + + +def test_artifact_store_roundtrip_bytes(monkeypatch, tmp_path): + store = ArtifactStore(str(tmp_path)) + target = store.path_for("org-a", "report-1", "report.md") + store.ensure_parent(target) + store.store_bytes(target, b"# title\nbody") + assert store.read_bytes(target) == b"# title\nbody" + assert store.exists(target) + store.delete(target) + assert not store.exists(target) + + +def test_default_artifact_store_uses_configured_upload_root(monkeypatch, tmp_path): + from app.config import Config + from app.services.artifact_store import default_artifact_store + + original = Config.UPLOAD_FOLDER + monkeypatch.setattr(Config, "UPLOAD_FOLDER", str(tmp_path / "uploads")) + try: + store = default_artifact_store() + project_dir = store.path_for("org-a", "project-1") + assert str(Config.UPLOAD_FOLDER) in project_dir + # Tenant-scoped and confinement still apply. + assert ".." not in project_dir + with pytest.raises(ValueError): + store.path_for("org-a", "..") + finally: + Config.UPLOAD_FOLDER = original diff --git a/backend/tests/test_audit_service.py b/backend/tests/test_audit_service.py new file mode 100644 index 0000000..3763885 --- /dev/null +++ b/backend/tests/test_audit_service.py @@ -0,0 +1,127 @@ +"""TDD gate: durable, redacted audit event recording. + +Audit events must be tenant-scoped and never store secrets, tokens, password +hashes, or raw prompts. This gate proves an AuditService that records actions +and lists them scoped to an organization. +""" + +import pytest +from sqlalchemy import create_engine + +from app.db import Base, create_session_factory +from app.services.audit_service import AuditService + + +@pytest.fixture() +def session_factory(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + try: + yield factory + finally: + engine.dispose() + + +def test_audit_service_records_and_lists_scoped(session_factory): + session = session_factory() + try: + svc = AuditService(session) + entry_id = svc.record( + organization_id="org-a", + actor_user_id="user-1", + action="user.role_changed", + target_type="user", + target_id="user-2", + details={"role": "admin"}, + ) + assert entry_id + + rows = svc.list_for_organization(organization_id="org-a") + assert len(rows) == 1 + assert rows[0].action == "user.role_changed" + assert rows[0].actor_user_id == "user-1" + + # Scoped list from another org does not see it. + other = svc.list_for_organization(organization_id="org-b") + assert other == [] + finally: + session.close() + + +def test_audit_service_redacts_secrets_from_details(session_factory): + session = session_factory() + try: + svc = AuditService(session) + svc.record( + organization_id="org-a", + actor_user_id="user-1", + action="auth.password_changed", + target_type="user", + target_id="user-2", + details={"password": "hunter2", "token": "abc", "api_key": "sk-xyz"}, + ) + row = svc.list_for_organization(organization_id="org-a")[0] + details = row.details if isinstance(row.details, dict) else {} + blob = repr(details) + assert "hunter2" not in blob + assert "abc" not in blob + assert "sk-xyz" not in blob + # Redaction leaves non-sensitive context intact. + assert details == {} + finally: + session.close() + + +def test_login_endpoint_writes_audit_event(tmp_path, monkeypatch, session_factory): + """A successful login records an auth.login audit event.""" + import secrets + + from flask import Flask, jsonify + + from app.api.auth import auth_bp + from app.services.audit_service import AuditService + from app.services.identity import IdentityRepository, PasswordService + from app.utils.api_errors import ApiError + from app.utils.locale import t + + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + + app = Flask(__name__) + app.config.update(TESTING=True, SESSION_COOKIE_SECURE=False) + app.config["SECRET_KEY"] = secrets.token_hex(32) + app.extensions["crowdsight_session_factory"] = factory + + @app.errorhandler(ApiError) + def handle_api_error(error): + return jsonify(error.to_payload(t)), error.status_code + + app.register_blueprint(auth_bp, url_prefix="/api/auth") + + audit_email = "audit-login@example.com" + with factory() as session: + repo = IdentityRepository(session) + org = repo.create_organization(name="Audit Org", slug="audit-org") + user = repo.create_user( + email=audit_email, + password_hash=PasswordService.hash_password("correct-horse"), + ) + repo.create_membership(user.id, org.id, "user") + session.commit() + user_id = user.id + org_id = org.id + + client = app.test_client() + r = client.post( + "/api/auth/login", + json={"email": audit_email, "password": "correct-horse"}, + ) + assert r.status_code == 200 + + # The audit event exists and is redacted/org-scoped. + with factory() as session: + events = AuditService(session).list_for_organization(organization_id=org_id) + assert any(e.action == "auth.login" and e.actor_user_id == user_id for e in events) + engine.dispose() diff --git a/backend/tests/test_auth_api.py b/backend/tests/test_auth_api.py new file mode 100644 index 0000000..9ed8cb1 --- /dev/null +++ b/backend/tests/test_auth_api.py @@ -0,0 +1,81 @@ +import json + +from flask import Flask +from sqlalchemy import create_engine +from sqlalchemy.orm import Session + +from app.api.auth import auth_bp +from app.db import Base, create_session_factory +from app.services.identity import IdentityRepository, PasswordService + + +def make_auth_app(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + + app = Flask(__name__) + app.config.update(TESTING=True, SECRET_KEY="test-secret", SESSION_COOKIE_SECURE=False) + app.extensions["crowdsight_session_factory"] = session_factory + app.register_blueprint(auth_bp, url_prefix="/api/auth") + + with session_factory() as session: + repo = IdentityRepository(session) + org = repo.create_organization(name="Org A", slug="org-a") + user = repo.create_user( + email="admin@example.com", + password_hash=PasswordService.hash_password("correct horse battery staple"), + ) + repo.create_membership(user.id, org.id, "admin") + session.commit() + + return app, engine + + +def _csrf_headers(client): + return {"X-CSRF-Token": client.get_cookie("crowdsight_csrf").value} + + +def test_login_me_logout_uses_cookie_and_allowlisted_identity(): + app, engine = make_auth_app() + try: + client = app.test_client() + login = client.post( + "/api/auth/login", + json={"email": "ADMIN@example.com", "password": "correct horse battery staple"}, + ) + assert login.status_code == 200 + body = login.get_json() + assert body["success"] is True + assert body["data"]["user"]["email"] == "admin@example.com" + assert body["data"]["role"] == "admin" + assert body["data"]["organization"]["slug"] == "org-a" + assert "token" not in json.dumps(body) + assert "password" not in json.dumps(body).lower() + + me = client.get("/api/auth/me") + assert me.status_code == 200 + assert me.get_json()["data"]["user"]["id"] + + logout = client.post("/api/auth/logout", headers=_csrf_headers(client)) + assert logout.status_code == 200 + assert client.get("/api/auth/me").status_code == 401 + finally: + engine.dispose() + + +def test_invalid_credentials_return_structured_generic_error(): + app, engine = make_auth_app() + try: + response = app.test_client().post( + "/api/auth/login", + json={"email": "admin@example.com", "password": "wrong password"}, + ) + assert response.status_code == 401 + body = response.get_json() + assert body["success"] is False + assert body["error_code"] == "invalid_credentials" + assert "error" not in body + assert "Traceback" not in json.dumps(body) + finally: + engine.dispose() diff --git a/backend/tests/test_auth_service.py b/backend/tests/test_auth_service.py new file mode 100644 index 0000000..84327d8 --- /dev/null +++ b/backend/tests/test_auth_service.py @@ -0,0 +1,87 @@ +from datetime import datetime, timedelta, timezone + +from app.models.saas import AuthSession +from app.services.identity import IdentityRepository, PasswordService, SessionService + + +def test_session_token_is_hashed_and_resolvable(): + from sqlalchemy import create_engine + from sqlalchemy.orm import Session + from app.db import Base + + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session = Session(engine) + try: + repo = IdentityRepository(session) + org = repo.create_organization(name="Org A", slug="org-a") + user = repo.create_user( + email="admin@example.com", + password_hash=PasswordService.hash_password("correct horse battery staple"), + ) + membership = repo.create_membership(user.id, org.id, "admin") + raw_token, stored = SessionService.create(session, user, membership.id) + session.commit() + + assert raw_token != stored.token_hash + assert len(stored.token_hash) == 64 + resolved = SessionService.resolve(session, raw_token) + assert resolved is not None + assert resolved.user.id == user.id + assert resolved.membership.id == membership.id + assert resolved.organization.id == org.id + finally: + session.close() + engine.dispose() + + +def test_revoked_or_stale_auth_version_session_is_not_resolvable(): + from sqlalchemy import create_engine + from sqlalchemy.orm import Session + from app.db import Base + + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session = Session(engine) + try: + repo = IdentityRepository(session) + org = repo.create_organization(name="Org A", slug="org-a") + user = repo.create_user(email="user@example.com") + membership = repo.create_membership(user.id, org.id, "user") + raw_token, _stored = SessionService.create(session, user, membership.id) + session.commit() + + user.auth_version += 1 + session.commit() + assert SessionService.resolve(session, raw_token) is None + + user.auth_version -= 1 + session.commit() + assert SessionService.revoke(session, raw_token) is True + session.commit() + assert SessionService.resolve(session, raw_token) is None + finally: + session.close() + engine.dispose() + + +def test_expired_session_is_not_resolvable(): + from sqlalchemy import create_engine + from sqlalchemy.orm import Session + from app.db import Base + + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session = Session(engine) + try: + repo = IdentityRepository(session) + org = repo.create_organization(name="Org A", slug="org-a") + user = repo.create_user(email="expired@example.com") + membership = repo.create_membership(user.id, org.id, "user") + raw_token, stored = SessionService.create(session, user, membership.id) + stored.expires_at = datetime.now(timezone.utc) - timedelta(seconds=1) + session.commit() + assert SessionService.resolve(session, raw_token) is None + finally: + session.close() + engine.dispose() diff --git a/backend/tests/test_authorization_policy.py b/backend/tests/test_authorization_policy.py new file mode 100644 index 0000000..5d4fca1 --- /dev/null +++ b/backend/tests/test_authorization_policy.py @@ -0,0 +1,126 @@ +import importlib.util +from pathlib import Path +import sys +import unittest + + +_MODULE_PATH = Path(__file__).parents[1] / "app" / "security" / "policy.py" +_SPEC = importlib.util.spec_from_file_location("authorization_policy_under_test", _MODULE_PATH) +assert _SPEC is not None and _SPEC.loader is not None +_POLICY = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _POLICY +_SPEC.loader.exec_module(_POLICY) + +Actor = _POLICY.Actor +AuthorizationError = _POLICY.AuthorizationError +Role = _POLICY.Role +assert_can_access_resource = _POLICY.assert_can_access_resource +assert_can_manage_llm_settings = _POLICY.assert_can_manage_llm_settings +assert_can_manage_user = _POLICY.assert_can_manage_user + + +class AuthorizationPolicyTests(unittest.TestCase): + def setUp(self): + self.admin = Actor(user_id="admin-1", organization_id="org-a", role=Role.ADMIN) + self.user = Actor(user_id="user-1", organization_id="org-a", role=Role.USER) + self.other_user = Actor(user_id="user-2", organization_id="org-a", role=Role.USER) + self.super_admin = Actor( + user_id="root-1", organization_id="platform", role=Role.SUPER_ADMIN + ) + + def test_user_can_access_owned_resource_in_own_organization(self): + assert_can_access_resource( + self.user, + resource_organization_id="org-a", + owner_user_id="user-1", + ) + + def test_user_cannot_access_another_users_resource_in_same_organization(self): + with self.assertRaises(AuthorizationError): + assert_can_access_resource( + self.user, + resource_organization_id="org-a", + owner_user_id="user-2", + ) + + def test_user_cannot_access_resource_from_another_organization(self): + with self.assertRaises(AuthorizationError): + assert_can_access_resource( + self.user, + resource_organization_id="org-b", + owner_user_id="user-1", + ) + + def test_admin_can_access_resources_in_own_organization(self): + assert_can_access_resource( + self.admin, + resource_organization_id="org-a", + owner_user_id="user-2", + ) + + def test_admin_cannot_cross_organization_boundary(self): + with self.assertRaises(AuthorizationError): + assert_can_access_resource( + self.admin, + resource_organization_id="org-b", + owner_user_id="user-9", + ) + + def test_super_admin_requires_explicit_platform_scope_for_cross_tenant_access(self): + with self.assertRaises(AuthorizationError): + assert_can_access_resource( + self.super_admin, + resource_organization_id="org-a", + owner_user_id="user-9", + ) + + assert_can_access_resource( + self.super_admin, + resource_organization_id="org-a", + owner_user_id="user-9", + platform_scope=True, + ) + + def test_admin_can_manage_user_but_not_grant_admin_or_super_admin(self): + assert_can_manage_user( + self.admin, + target_organization_id="org-a", + target_role=Role.USER, + ) + with self.assertRaises(AuthorizationError): + assert_can_manage_user( + self.admin, + target_organization_id="org-a", + target_role=Role.ADMIN, + ) + with self.assertRaises(AuthorizationError): + assert_can_manage_user( + self.admin, + target_organization_id="org-a", + target_role=Role.SUPER_ADMIN, + ) + + def test_super_admin_can_manage_any_role_only_with_platform_scope(self): + with self.assertRaises(AuthorizationError): + assert_can_manage_user( + self.super_admin, + target_organization_id="org-a", + target_role=Role.ADMIN, + ) + assert_can_manage_user( + self.super_admin, + target_organization_id="org-a", + target_role=Role.ADMIN, + platform_scope=True, + ) + + def test_only_super_admin_can_manage_llm_settings(self): + with self.assertRaises(AuthorizationError): + assert_can_manage_llm_settings(self.admin) + with self.assertRaises(AuthorizationError): + assert_can_manage_llm_settings(self.user) + assert_can_manage_llm_settings(self.super_admin, platform_scope=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/tests/test_auxiliary_api_security.py b/backend/tests/test_auxiliary_api_security.py new file mode 100644 index 0000000..962ccd5 --- /dev/null +++ b/backend/tests/test_auxiliary_api_security.py @@ -0,0 +1,249 @@ +from __future__ import annotations + +import importlib +import secrets + +import pytest +from flask import Flask, jsonify +from sqlalchemy import create_engine + +from app.api.agent_group import agent_group_bp +from app.api.auth import auth_bp +from app.api.template import template_bp +from app.db import Base, create_session_factory +from app.services.identity import IdentityRepository, PasswordService +from app.utils.api_errors import ApiError +from app.utils.locale import t + +agent_group_module = importlib.import_module("app.api.agent_group") +template_module = importlib.import_module("app.api.template") + + +TEST_EMAIL = "auxiliary-security@example.com" +TEST_AUTH_INPUT = "local-only-auth-input" + + +def make_app(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + + app = Flask(__name__) + app.config.update( + TESTING=True, + SESSION_COOKIE_SECURE=False, + ) + app.config["SECRET_KEY"] = secrets.token_hex(32) + app.extensions["crowdsight_session_factory"] = session_factory + + @app.errorhandler(ApiError) + def handle_api_error(error: ApiError): + return jsonify(error.to_payload(t)), error.status_code + + app.register_blueprint(auth_bp, url_prefix="/api/auth") + app.register_blueprint(template_bp, url_prefix="/api/template") + app.register_blueprint(agent_group_bp, url_prefix="/api/agent-group") + + with session_factory() as session: + repo = IdentityRepository(session) + organization = repo.create_organization(name="Auxiliary Org", slug="auxiliary-org") + user = repo.create_user( + email=TEST_EMAIL, + password_hash=PasswordService.hash_password(TEST_AUTH_INPUT), + ) + repo.create_membership(user.id, organization.id, "admin") + session.commit() + + return app, engine + + +def login(client): + response = client.post( + "/api/auth/login", + json={"email": TEST_EMAIL, "password": TEST_AUTH_INPUT}, + ) + assert response.status_code == 200 + return client.get_cookie("crowdsight_csrf").value + + +def test_template_list_requires_authentication(): + app, engine = make_app() + try: + response = app.test_client().get("/api/template/list") + assert response.status_code == 401 + assert response.get_json()["error_code"] == "unauthorized" + finally: + engine.dispose() + + +def test_agent_group_categorize_requires_authentication(): + app, engine = make_app() + try: + response = app.test_client().post( + "/api/agent-group/categorize", + json={"agents": [{"name": "Alice"}]}, + ) + assert response.status_code == 401 + assert response.get_json()["error_code"] == "unauthorized" + finally: + engine.dispose() + + +def test_authenticated_auxiliary_reads_and_pure_filter_remain_available(): + app, engine = make_app() + try: + client = app.test_client() + csrf = login(client) + + templates = client.get("/api/template/list") + assert templates.status_code == 200 + assert templates.get_json()["success"] is True + + filtered = client.post( + "/api/agent-group/filter", + json={ + "agents": [{"agent_id": 1}], + "groups": [{"group_id": "all", "agent_indices": [0]}], + "selected_group_ids": ["all"], + }, + headers={"X-CSRF-Token": csrf}, + ) + assert filtered.status_code == 200 + assert filtered.get_json()["selected_agent_ids"] == [0] + finally: + engine.dispose() + + +@pytest.mark.parametrize( + ("path", "payload"), + [ + ("/api/template/auto-select", {"text": "Alice founded Orbit."}), + ("/api/agent-group/categorize", {"agents": [{"name": "Alice"}]}), + ], +) +def test_auxiliary_llm_mutations_require_csrf_and_idempotency(path, payload): + app, engine = make_app() + try: + client = app.test_client() + csrf = login(client) + + missing_csrf = client.post( + path, + json=payload, + headers={"Idempotency-Key": "auxiliary-mutation-1"}, + ) + assert missing_csrf.status_code == 403 + assert missing_csrf.get_json()["error_code"] == "csrf_failed" + + missing_idempotency = client.post( + path, + json=payload, + headers={"X-CSRF-Token": csrf}, + ) + assert missing_idempotency.status_code == 400 + assert missing_idempotency.get_json()["error_code"] == "idempotency_required" + finally: + engine.dispose() + + +def test_every_auxiliary_route_requires_authentication(): + app, engine = make_app() + try: + client = app.test_client() + routes = [ + ("GET", "/api/template/list", None), + ("GET", "/api/template/news_event/filter-rules", None), + ("POST", "/api/template/auto-select", {"text": "Alice founded Orbit."}), + ("POST", "/api/agent-group/filter", {"agents": [], "groups": []}), + ("POST", "/api/agent-group/categorize", {"agents": [{"name": "Alice"}]}), + ] + + for method, path, payload in routes: + response = client.open(path, method=method, json=payload) + assert response.status_code == 401, (method, path, response.get_json()) + finally: + engine.dispose() + + +def test_llm_auxiliary_errors_are_safe(monkeypatch): + class ExplodingLLM: + def chat_json(self, **_kwargs): + raise RuntimeError("sensitive backend detail") + + monkeypatch.setattr(template_module, "LLMClient", ExplodingLLM) + monkeypatch.setattr(agent_group_module, "LLMClient", ExplodingLLM) + app, engine = make_app() + try: + client = app.test_client() + csrf = login(client) + responses = [ + client.post( + "/api/template/auto-select", + json={"text": "Alice founded Orbit."}, + headers={"X-CSRF-Token": csrf, "Idempotency-Key": "safe-template-error"}, + ), + client.post( + "/api/agent-group/categorize", + json={"agents": [{"name": "Alice"}]}, + headers={"X-CSRF-Token": csrf, "Idempotency-Key": "safe-agent-error"}, + ), + ] + for response in responses: + body = response.get_json() + assert response.status_code == 500 + assert body["error_code"] == "internal_error" + assert "sensitive backend detail" not in response.get_data(as_text=True) + finally: + engine.dispose() + + +def test_llm_auxiliary_mutations_replay_completed_responses(monkeypatch): + class FakeTemplateLLM: + def chat_json(self, **_kwargs): + return { + "template_id": "news_event", + "prompt": "Alice founded Orbit.", + "confidence": 0.9, + "reasoning": "The input describes a news event.", + } + + class FakeAgentLLM: + def chat_json(self, **_kwargs): + return { + "groups": [ + { + "group_id": "audience", + "group_name": "Audience", + "default_enabled": True, + "agent_indices": [0], + } + ] + } + + monkeypatch.setattr(template_module, "LLMClient", FakeTemplateLLM) + monkeypatch.setattr(agent_group_module, "LLMClient", FakeAgentLLM) + app, engine = make_app() + try: + client = app.test_client() + csrf = login(client) + cases = [ + ( + "/api/template/auto-select", + {"text": "Alice founded Orbit."}, + "template-replay", + ), + ( + "/api/agent-group/categorize", + {"agents": [{"name": "Alice"}]}, + "agent-replay", + ), + ] + for path, payload, key in cases: + headers = {"X-CSRF-Token": csrf, "Idempotency-Key": key} + first = client.post(path, json=payload, headers=headers) + second = client.post(path, json=payload, headers=headers) + assert first.status_code == 200 + assert second.status_code == 200 + assert second.get_json() == first.get_json() + finally: + engine.dispose() diff --git a/backend/tests/test_child_resource_scope.py b/backend/tests/test_child_resource_scope.py new file mode 100644 index 0000000..affa236 --- /dev/null +++ b/backend/tests/test_child_resource_scope.py @@ -0,0 +1,41 @@ +from tempfile import TemporaryDirectory + +from app.models.project import ProjectManager +from app.services.simulation_manager import SimulationManager +from test_resource_auth_scope import csrf_headers, login, make_resource_app + + +def test_simulation_create_rejects_in_scope_graph_from_another_project(): + app, engine, organization_id, user_a_id, _user_b_id = make_resource_app() + original_projects_dir = ProjectManager.PROJECTS_DIR + original_simulations_dir = SimulationManager.SIMULATION_DATA_DIR + try: + with TemporaryDirectory() as temp_dir: + ProjectManager.PROJECTS_DIR = f"{temp_dir}/projects" + SimulationManager.SIMULATION_DATA_DIR = f"{temp_dir}/simulations" + project_a = ProjectManager.create_project( + "A", organization_id=organization_id, owner_user_id=user_a_id + ) + project_a.graph_id = "graph-a" + ProjectManager.save_project(project_a) + project_b = ProjectManager.create_project( + "B", organization_id=organization_id, owner_user_id=user_a_id + ) + project_b.graph_id = "graph-b" + ProjectManager.save_project(project_b) + + client = app.test_client() + login(client) + response = client.post( + "/api/simulation/create", + json={"project_id": project_a.project_id, "graph_id": project_b.graph_id}, + headers={ + **csrf_headers(client), + "Idempotency-Key": "cross-project-graph", + }, + ) + assert response.status_code == 404 + finally: + ProjectManager.PROJECTS_DIR = original_projects_dir + SimulationManager.SIMULATION_DATA_DIR = original_simulations_dir + engine.dispose() diff --git a/backend/tests/test_cors_config.py b/backend/tests/test_cors_config.py new file mode 100644 index 0000000..6684a7b --- /dev/null +++ b/backend/tests/test_cors_config.py @@ -0,0 +1,43 @@ +import os + +import pytest + +from app import create_app +from app.config import Config +from app.models.task import TaskManager + + +class TestConfig(Config): + SECRET_KEY = "test-secret" + MEMORY_BACKEND = "local" + CORS_ALLOWED_ORIGINS = ["https://allowed.example"] + TESTING = True + + +def test_cors_uses_app_allowlist_and_credentials(): + previous = os.environ.get("DATABASE_URL") + os.environ["DATABASE_URL"] = "sqlite+pysqlite:///:memory:" + try: + app = create_app(TestConfig) + response = app.test_client().options( + "/api/graph/project/list", + headers={ + "Origin": "https://allowed.example", + "Access-Control-Request-Method": "GET", + }, + ) + assert response.headers["Access-Control-Allow-Origin"] == "https://allowed.example" + assert response.headers["Access-Control-Allow-Credentials"] == "true" + finally: + if previous is None: + os.environ.pop("DATABASE_URL", None) + else: + os.environ["DATABASE_URL"] = previous + + +def test_cors_wildcard_is_rejected_with_cookie_auth(): + class WildcardConfig(TestConfig): + CORS_ALLOWED_ORIGINS = ["*"] + + with pytest.raises(RuntimeError, match="wildcard_cors_not_allowed"): + create_app(WildcardConfig) diff --git a/backend/tests/test_durable_task_manager.py b/backend/tests/test_durable_task_manager.py new file mode 100644 index 0000000..9d9e967 --- /dev/null +++ b/backend/tests/test_durable_task_manager.py @@ -0,0 +1,209 @@ +import pytest +from flask import Flask +from sqlalchemy import create_engine + +from app.db import Base, create_session_factory +from app.models.task import TaskManager, TaskStatus +from app.security.policy import Role +from app.services.identity import IdentityRepository + + +def test_task_state_survives_task_manager_reconfiguration(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + try: + with factory() as session: + repo = IdentityRepository(session) + org = repo.create_organization(name="Jobs", slug="jobs-org") + user = repo.create_user(email="jobs@example.com") + repo.create_membership(user.id, org.id, Role.ADMIN) + session.commit() + + TaskManager.configure(factory) + first_manager = TaskManager() + task_id = first_manager.create_task( + "durable_test", + metadata={"organization_id": org.id, "owner_user_id": user.id}, + ) + first_manager.update_task( + task_id, + status=TaskStatus.PROCESSING, + progress=42, + message="working", + progress_detail={"stage": "one"}, + ) + + TaskManager.configure(factory) + restarted_manager = TaskManager() + task = restarted_manager.get_task(task_id) + assert task is not None + assert task.status is TaskStatus.PROCESSING + assert task.progress == 42 + assert task.message == "working" + assert task.progress_detail == {"stage": "one"} + finally: + TaskManager.configure(None) + engine.dispose() + + +def test_task_manager_query_filters_are_tenant_scoped(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + try: + TaskManager.configure(factory) + manager = TaskManager() + task_a = manager.create_task("alpha", {"organization_id": "org-a", "owner_user_id": "user-a"}) + task_b = manager.create_task("beta", {"organization_id": "org-b", "owner_user_id": "user-b"}) + + assert manager.get_task(task_a, organization_id="org-a") is not None + assert manager.get_task(task_b, organization_id="org-a") is None + assert manager.get_task(task_a, organization_id="org-a", owner_user_id="user-b") is None + assert [task.metadata["organization_id"] for task in manager.list_tasks(organization_id="org-a")] == ["org-a"] + assert [task.metadata["owner_user_id"] for task in manager.list_tasks(organization_id="org-a", owner_user_id="user-a")] == ["user-a"] + assert manager.list_tasks(organization_id="org-a", owner_user_id="user-b") == [] + finally: + TaskManager.configure(None) + engine.dispose() + + +def test_task_manager_fails_closed_inside_app_without_session_factory(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + app = Flask("missing-session-factory") + try: + TaskManager.configure(factory) + with app.app_context(): + with pytest.raises(RuntimeError, match="task_session_factory_required"): + TaskManager() + finally: + TaskManager.configure(None) + engine.dispose() + + +def test_task_manager_binds_to_current_app_without_cross_app_leak(): + engines = [] + apps = [] + organization_ids = [] + try: + for slug in ("app-a", "app-b"): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + with factory() as session: + repo = IdentityRepository(session) + organization = repo.create_organization(name=slug, slug=slug) + session.commit() + app = Flask(slug) + app.extensions["crowdsight_session_factory"] = factory + engines.append(engine) + apps.append(app) + organization_ids.append(organization.id) + + TaskManager.configure(None) + with apps[0].app_context(): + task_a = TaskManager().create_task( + "app_a_task", + metadata={"organization_id": organization_ids[0]}, + ) + with apps[1].app_context(): + assert TaskManager().get_task(task_a) is None + task_b = TaskManager().create_task( + "app_b_task", + metadata={"organization_id": organization_ids[1]}, + ) + with apps[0].app_context(): + assert TaskManager().get_task(task_a) is not None + assert TaskManager().get_task(task_b) is None + finally: + TaskManager.configure(None) + for engine in engines: + engine.dispose() + + +def test_reused_task_manager_fails_closed_across_app_contexts(): + engines = [] + apps = [] + organization_ids = [] + try: + for slug in ("reuse-a", "reuse-b"): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + with factory() as session: + repo = IdentityRepository(session) + organization = repo.create_organization(name=slug, slug=slug) + session.commit() + app = Flask(slug) + app.extensions["crowdsight_session_factory"] = factory + engines.append(engine) + apps.append(app) + organization_ids.append(organization.id) + + with apps[0].app_context(): + manager = TaskManager() + task_id = manager.create_task("reuse_task", {"organization_id": organization_ids[0]}) + with apps[1].app_context(): + with pytest.raises(RuntimeError, match="task_app_context_mismatch"): + manager.get_task(task_id) + finally: + TaskManager.configure(None) + for engine in engines: + engine.dispose() + + +def test_explicit_task_manager_factory_must_match_current_app(): + engine_a = create_engine("sqlite+pysqlite:///:memory:") + engine_b = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine_a) + Base.metadata.create_all(engine_b) + factory_a = create_session_factory(engine_a) + factory_b = create_session_factory(engine_b) + app_b = Flask("explicit-b") + app_b.extensions["crowdsight_session_factory"] = factory_b + try: + with app_b.app_context(): + with pytest.raises(RuntimeError, match="task_session_factory_mismatch"): + TaskManager(session_factory=factory_a) + finally: + TaskManager.configure(None) + engine_a.dispose() + engine_b.dispose() + + +def test_prebound_task_manager_fails_closed_inside_other_app(): + engine_a = create_engine("sqlite+pysqlite:///:memory:") + engine_b = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine_a) + Base.metadata.create_all(engine_b) + factory_a = create_session_factory(engine_a) + factory_b = create_session_factory(engine_b) + app_b = Flask("prebound-b") + app_b.extensions["crowdsight_session_factory"] = factory_b + manager = TaskManager(session_factory=factory_a) + try: + with app_b.app_context(): + with pytest.raises(RuntimeError, match="task_session_factory_mismatch"): + manager.get_task("not-in-scope") + finally: + TaskManager.configure(None) + engine_a.dispose() + engine_b.dispose() + + +def test_app_bound_task_manager_supports_background_use_without_context(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + app = Flask("background-use") + app.extensions["crowdsight_session_factory"] = factory + try: + with app.app_context(): + manager = TaskManager() + task_id = manager.create_task("background_task", {"organization_id": "org-bg"}) + assert manager.get_task(task_id) is not None + finally: + TaskManager.configure(None) + engine.dispose() diff --git a/backend/tests/test_graph_auth_scope.py b/backend/tests/test_graph_auth_scope.py new file mode 100644 index 0000000..937477d --- /dev/null +++ b/backend/tests/test_graph_auth_scope.py @@ -0,0 +1,101 @@ +import tempfile +from pathlib import Path + +from flask import Flask +from sqlalchemy import create_engine + +from app.api import graph_bp +from app.api.auth import auth_bp +from app.db import Base, create_session_factory +from app.models.project import ProjectManager +from app.models.saas import Organization +from app.services.identity import IdentityRepository, PasswordService + + +def make_graph_app(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + app = Flask(__name__) + app.config.update(TESTING=True, SECRET_KEY="test-secret", SESSION_COOKIE_SECURE=False) + app.extensions["crowdsight_session_factory"] = session_factory + app.register_blueprint(auth_bp, url_prefix="/api/auth") + app.register_blueprint(graph_bp, url_prefix="/api/graph") + + with session_factory() as session: + repo = IdentityRepository(session) + org = repo.create_organization(name="Org A", slug="org-a") + user_a = repo.create_user( + email="user-a@example.com", + password_hash=PasswordService.hash_password("correct horse battery staple"), + ) + repo.create_membership(user_a.id, org.id, "user") + user_b = repo.create_user( + email="user-b@example.com", + password_hash=PasswordService.hash_password("correct horse battery staple"), + ) + repo.create_membership(user_b.id, org.id, "user") + admin = repo.create_user( + email="admin@example.com", + password_hash=PasswordService.hash_password("correct horse battery staple"), + ) + repo.create_membership(admin.id, org.id, "admin") + session.commit() + + return app, engine, user_a.id, user_b.id + + +def login(client, email): + response = client.post( + "/api/auth/login", + json={"email": email, "password": "correct horse battery staple"}, + ) + assert response.status_code == 200 + + +def test_graph_data_and_task_routes_require_auth(): + original_dir = ProjectManager.PROJECTS_DIR + app, engine, _user_a_id, _user_b_id = make_graph_app() + with tempfile.TemporaryDirectory() as temp_dir: + ProjectManager.PROJECTS_DIR = str(Path(temp_dir) / "projects") + try: + client = app.test_client() + assert client.get("/api/graph/data/graph-any").status_code == 401 + login(client, "user-a@example.com") + assert client.get("/api/graph/data/arbitrary-graph").status_code == 404 + assert client.get("/api/graph/task/arbitrary-task").status_code == 404 + finally: + ProjectManager.PROJECTS_DIR = original_dir + engine.dispose() + + +def test_graph_project_routes_require_auth_and_scope_records(): + original_dir = ProjectManager.PROJECTS_DIR + app, engine, user_a_id, user_b_id = make_graph_app() + with tempfile.TemporaryDirectory() as temp_dir: + ProjectManager.PROJECTS_DIR = str(Path(temp_dir) / "projects") + try: + project_a = ProjectManager.create_project( + "A", organization_id="org_a_placeholder", owner_user_id=user_a_id + ) + project_b = ProjectManager.create_project( + "B", organization_id="org_a_placeholder", owner_user_id=user_b_id + ) + # The app's organization id is discovered from the seeded user. + with app.extensions["crowdsight_session_factory"]() as session: + org_id = session.query(Organization).one().id + for project in (project_a, project_b): + project.organization_id = org_id + ProjectManager.save_project(project) + + client = app.test_client() + assert client.get(f"/api/graph/project/{project_a.project_id}").status_code == 401 + + login(client, "user-a@example.com") + assert client.get(f"/api/graph/project/{project_a.project_id}").status_code == 200 + assert client.get(f"/api/graph/project/{project_b.project_id}").status_code == 404 + listed = client.get("/api/graph/project/list").get_json() + assert [item["project_id"] for item in listed["data"]] == [project_a.project_id] + finally: + ProjectManager.PROJECTS_DIR = original_dir + engine.dispose() diff --git a/backend/tests/test_graph_backend_routes.py b/backend/tests/test_graph_backend_routes.py new file mode 100644 index 0000000..ae5c22f --- /dev/null +++ b/backend/tests/test_graph_backend_routes.py @@ -0,0 +1,201 @@ +from types import SimpleNamespace + +import pytest +from flask import Flask, g + +import app.api.graph as graph_api +from app.api import graph_bp +from app.config import Config +from app.models.project import ProjectManager +from app.security.policy import Role +from app.services.local_graph_builder import LocalGraphBuilderService + + +class _LocalGraphSpy: + instances = [] + + def __init__(self, session_factory, **kwargs): + self.session_factory = session_factory + self.kwargs = kwargs + self.calls = [] + self.__class__.instances.append(self) + + def get_graph_data(self, graph_id): + self.calls.append(("get_graph_data", graph_id)) + return {"graph_id": graph_id, "node_count": 0, "edge_count": 0, "nodes": [], "edges": []} + + def delete_graph(self, graph_id): + self.calls.append(("delete_graph", graph_id)) + + +def make_app(): + app = Flask(__name__) + app.config.update(TESTING=True, SECRET_KEY="unit-test") + app.extensions["crowdsight_session_factory"] = lambda: None + app.register_blueprint(graph_bp, url_prefix="/api/graph") + return app + + +def set_actor(): + g.auth_context = SimpleNamespace( + user=SimpleNamespace(id="user-a"), + membership=SimpleNamespace(role=Role.USER), + organization=SimpleNamespace(id="org-a"), + ) + + +def test_delete_project_rejects_out_of_scope_project_before_delete(monkeypatch): + app = make_app() + delete_calls = [] + monkeypatch.setattr(graph_api, "_scoped_project", lambda project_id: None) + monkeypatch.setattr( + ProjectManager, + "delete_project", + lambda project_id, **kwargs: delete_calls.append((project_id, kwargs)) or True, + ) + + with app.test_request_context("/api/graph/project/project-b", method="DELETE"): + set_actor() + result = graph_api.delete_project.__wrapped__("project-b") + + if isinstance(result, tuple): + response, status = result + else: + response, status = result, 200 + assert status == 404 + assert response.get_json()["success"] is False + assert delete_calls == [] + + +def test_local_graph_data_route_uses_local_adapter_without_constructing_zep(monkeypatch): + app = make_app() + old_backend = Config.MEMORY_BACKEND + old_key = Config.ZEP_API_KEY + try: + monkeypatch.setattr(Config, "MEMORY_BACKEND", "local") + monkeypatch.setattr(Config, "ZEP_API_KEY", None) + monkeypatch.setattr( + graph_api, + "_scoped_graph", + lambda graph_id: SimpleNamespace(project_id="project-local"), + ) + monkeypatch.setattr(graph_api, "LocalGraphBuilderService", _LocalGraphSpy) + monkeypatch.setattr( + graph_api, + "GraphBuilderService", + lambda **kwargs: pytest.fail("local graph read must not construct Zep"), + ) + _LocalGraphSpy.instances.clear() + + with app.test_request_context("/api/graph/data/graph-local"): + set_actor() + response = graph_api.get_graph_data.__wrapped__("graph-local") + + if isinstance(response, tuple): + assert response[1] == 200 + payload = response[0].get_json() + else: + payload = response.get_json() + assert payload["success"] is True + assert payload["data"]["graph_id"] == "graph-local" + assert _LocalGraphSpy.instances[0].kwargs["organization_id"] == "org-a" + assert _LocalGraphSpy.instances[0].kwargs["project_id"] == "project-local" + assert _LocalGraphSpy.instances[0].calls == [("get_graph_data", "graph-local")] + finally: + Config.MEMORY_BACKEND = old_backend + Config.ZEP_API_KEY = old_key + + +def test_local_graph_delete_route_uses_local_adapter_without_constructing_zep(monkeypatch): + app = make_app() + old_backend = Config.MEMORY_BACKEND + old_key = Config.ZEP_API_KEY + try: + monkeypatch.setattr(Config, "MEMORY_BACKEND", "local") + monkeypatch.setattr(Config, "ZEP_API_KEY", None) + monkeypatch.setattr( + graph_api, + "_scoped_graph", + lambda graph_id: SimpleNamespace(project_id="project-local"), + ) + monkeypatch.setattr(graph_api, "LocalGraphBuilderService", _LocalGraphSpy) + monkeypatch.setattr( + graph_api, + "GraphBuilderService", + lambda **kwargs: pytest.fail("local graph delete must not construct Zep"), + ) + _LocalGraphSpy.instances.clear() + + with app.test_request_context("/api/graph/delete/graph-local", method="DELETE"): + set_actor() + response = graph_api.delete_graph.__wrapped__("graph-local") + + payload = response[0].get_json() if isinstance(response, tuple) else response.get_json() + assert payload["success"] is True + assert _LocalGraphSpy.instances[0].calls == [("delete_graph", "graph-local")] + finally: + Config.MEMORY_BACKEND = old_backend + Config.ZEP_API_KEY = old_key + + +def test_local_graph_builder_deletes_only_the_scoped_graph(): + from sqlalchemy import create_engine + from app.db import Base, create_session_factory + from app.models.memory import MemoryEdge, MemoryGraph, MemoryNode + + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + try: + builder = LocalGraphBuilderService( + session_factory, + organization_id="org-a", + project_id="project-a", + extraction_client=object(), + ) + graph_id = builder.create_graph(name="Local") + + with session_factory() as session: + first = MemoryNode( + id="node-a", + graph_id=graph_id, + canonical_name="A", + normalized_name="a", + ) + second = MemoryNode( + id="node-b", + graph_id=graph_id, + canonical_name="B", + normalized_name="b", + ) + session.add_all([first, second]) + session.flush() + session.add( + MemoryEdge( + id="edge-ab", + graph_id=graph_id, + source_node_id=first.id, + target_node_id=second.id, + relation="RELATED", + fact="A is related to B.", + ) + ) + session.commit() + + other_builder = LocalGraphBuilderService( + session_factory, + organization_id="org-b", + project_id="project-b", + extraction_client=object(), + ) + with pytest.raises(ValueError, match="memory_graph_not_found"): + other_builder.delete_graph(graph_id) + + builder.delete_graph(graph_id) + with session_factory() as session: + assert session.get(MemoryGraph, graph_id) is None + assert session.get(MemoryNode, "node-a") is None + assert session.get(MemoryNode, "node-b") is None + assert session.get(MemoryEdge, "edge-ab") is None + finally: + engine.dispose() diff --git a/backend/tests/test_graph_builder_task_scope.py b/backend/tests/test_graph_builder_task_scope.py new file mode 100644 index 0000000..12c82ce --- /dev/null +++ b/backend/tests/test_graph_builder_task_scope.py @@ -0,0 +1,49 @@ +from app.db import Base, create_database_engine, create_session_factory +from app.models.task import TaskManager +from app.services import graph_builder +from app.services.identity import IdentityRepository + + +def test_graph_builder_async_task_keeps_organization_scope(monkeypatch): + engine = create_database_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + + class FakeZep: + def __init__(self, api_key): + self.api_key = api_key + + class FakeThread: + def __init__(self, *, target, args): + self.target = target + self.args = args + self.daemon = False + + def start(self): + return None + + monkeypatch.setattr(graph_builder, "Zep", FakeZep) + monkeypatch.setattr(graph_builder.threading, "Thread", FakeThread) + + try: + with factory() as session: + organization = IdentityRepository(session).create_organization( + name="Graph Builder Tests", slug="graph-builder-tests" + ) + session.commit() + + builder = graph_builder.GraphBuilderService( + api_key="unit-test-placeholder", + organization_id=organization.id, + session_factory=factory, + ) + task_id = builder.build_graph_async("seed text", {"entity_types": []}) + + task = TaskManager(session_factory=factory).get_task( + task_id, organization_id=organization.id + ) + assert task is not None + assert task.metadata["organization_id"] == organization.id + finally: + TaskManager.configure(None) + engine.dispose() diff --git a/backend/tests/test_idempotency_api.py b/backend/tests/test_idempotency_api.py new file mode 100644 index 0000000..7ce69e6 --- /dev/null +++ b/backend/tests/test_idempotency_api.py @@ -0,0 +1,184 @@ +import secrets + +from flask import Flask, jsonify +from sqlalchemy import create_engine + +from app.db import Base, create_session_factory +from app.security.auth import require_auth +from app.security.policy import Role +from app.services.idempotency import IdempotencyService, _request_fingerprint_payload, idempotent +from app.services.identity import IdentityRepository, PasswordService +from app.utils.api_errors import ApiError +from app.utils.locale import t + + +def make_app(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + app = Flask(__name__) + app.config.update(TESTING=True, SESSION_COOKIE_SECURE=False) + app.config["SECRET_KEY"] = secrets.token_hex(32) + app.extensions["crowdsight_session_factory"] = session_factory + + @app.errorhandler(ApiError) + def handle_api_error(error): + return {**error.to_payload(t)}, error.status_code + + @app.post("/mutate") + @require_auth + @idempotent + def mutate(): + return jsonify({"success": True, "data": {"value": "created"}}), 201 + + @app.post("/mutate-alt") + @require_auth + @idempotent + def mutate_alt(): + return jsonify({"success": True, "data": {"value": "alternate"}}), 201 + + with session_factory() as session: + repo = IdentityRepository(session) + org = repo.create_organization(name="Org A", slug="idempotent-org") + user = repo.create_user( + email="idempotent@example.com", + password_hash=PasswordService.hash_password("correct horse battery staple"), + ) + repo.create_membership(user.id, org.id, Role.ADMIN) + session.commit() + return app, engine + + +def login(client): + response = client.post( + "/mutate", + json={"value": "created"}, + headers={"Idempotency-Key": "mutation-1"}, + ) + return response + + +def test_idempotent_route_replays_completed_response(): + app, engine = make_app() + try: + client = app.test_client() + auth_app = Flask(__name__) + # Login through the real blueprint in a small shared app is covered separately; + # seed the opaque cookie via the auth endpoint mounted on this test app. + from app.api.auth import auth_bp + app.register_blueprint(auth_bp, url_prefix="/api/auth") + login_response = client.post( + "/api/auth/login", + json={"email": "idempotent@example.com", "password": "correct horse battery staple"}, + ) + assert login_response.status_code == 200 + csrf = client.get_cookie("crowdsight_csrf").value + headers = {"Idempotency-Key": "mutation-1", "X-CSRF-Token": csrf} + + first = client.post("/mutate", json={"value": "created"}, headers=headers) + second = client.post("/mutate", json={"value": "created"}, headers=headers) + assert first.status_code == second.status_code == 201 + assert first.get_json() == second.get_json() + finally: + engine.dispose() + + +def test_idempotent_route_rejects_same_key_for_different_body(): + app, engine = make_app() + try: + client = app.test_client() + from app.api.auth import auth_bp + app.register_blueprint(auth_bp, url_prefix="/api/auth") + assert client.post( + "/api/auth/login", + json={"email": "idempotent@example.com", "password": "correct horse battery staple"}, + ).status_code == 200 + csrf = client.get_cookie("crowdsight_csrf").value + headers = {"Idempotency-Key": "mutation-2", "X-CSRF-Token": csrf} + assert client.post("/mutate", json={"value": "a"}, headers=headers).status_code == 201 + conflict = client.post("/mutate", json={"value": "b"}, headers=headers) + assert conflict.status_code == 409 + assert conflict.get_json()["error_code"] == "idempotency_key_reused" + finally: + engine.dispose() + + +def test_multipart_file_content_affects_fingerprint(): + app = Flask(__name__) + boundary = b"FixedBoundary" + + def fingerprint(content): + body = ( + b"--" + boundary + b"\r\n" + b'Content-Disposition: form-data; name="simulation_requirement"\r\n\r\n' + b"req\r\n" + b"--" + boundary + b"\r\n" + b'Content-Disposition: form-data; name="files"; filename="doc.txt"\r\n' + b"Content-Type: text/plain\r\n\r\n" + + content + + b"\r\n--" + + boundary + + b"--\r\n" + ) + with app.test_request_context( + "/upload", + method="POST", + data=body, + content_type="multipart/form-data; boundary=FixedBoundary", + ): + payload = _request_fingerprint_payload() + return IdempotencyService._request_hash(payload) + + assert fingerprint(b"alpha") != fingerprint(b"bravo") + + +def test_all_multipart_files_affect_fingerprint(): + app = Flask(__name__) + boundary = b"FixedBoundary" + + def fingerprint(second_content): + first_part = ( + b'Content-Disposition: form-data; name="files"; filename="first.txt"\r\n' + b"Content-Type: text/plain\r\n\r\nalpha" + ) + second_part = ( + b'Content-Disposition: form-data; name="files"; filename="second.txt"\r\n' + b"Content-Type: text/plain\r\n\r\n" + + second_content + ) + body = ( + b"--" + boundary + b"\r\n" + + first_part + + b"\r\n--" + boundary + b"\r\n" + + second_part + + b"\r\n--" + boundary + b"--\r\n" + ) + with app.test_request_context( + "/upload", + method="POST", + data=body, + content_type="multipart/form-data; boundary=FixedBoundary", + ): + return IdempotencyService._request_hash(_request_fingerprint_payload()) + + assert fingerprint(b"bravo") != fingerprint(b"charl") + + +def test_idempotency_key_cannot_replay_across_routes(): + app, engine = make_app() + try: + client = app.test_client() + from app.api.auth import auth_bp + app.register_blueprint(auth_bp, url_prefix="/api/auth") + assert client.post( + "/api/auth/login", + json={"email": "idempotent@example.com", "password": "correct horse battery staple"}, + ).status_code == 200 + csrf = client.get_cookie("crowdsight_csrf").value + headers = {"Idempotency-Key": "cross-route", "X-CSRF-Token": csrf} + assert client.post("/mutate", json={"value": "created"}, headers=headers).status_code == 201 + conflict = client.post("/mutate-alt", json={"value": "created"}, headers=headers) + assert conflict.status_code == 409 + assert conflict.get_json()["error_code"] == "idempotency_key_reused" + finally: + engine.dispose() diff --git a/backend/tests/test_job_queue.py b/backend/tests/test_job_queue.py new file mode 100644 index 0000000..52b009f --- /dev/null +++ b/backend/tests/test_job_queue.py @@ -0,0 +1,161 @@ +"""TDD gate: durable job queue claiming/consumption without a broker. + +A real worker later runs on a queue provider, but the claim/complete lifecycle +and tenant scope must work against the durable ``jobs`` table now so jobs +survive restarts and are isolated per organization. +""" + +import pytest +from sqlalchemy import create_engine + +from app.db import Base, create_session_factory +from app.models.operations import Job, JobStatus +from app.services.job_queue import JobQueue + + +@pytest.fixture() +def session_factory(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + try: + yield factory + finally: + engine.dispose() + + +def _seed_job(session_factory, *, organization_id="org-a", operation="graph.build"): + session = session_factory() + job = Job( + organization_id=organization_id, + owner_user_id="user-1", + operation=operation, + status=JobStatus.QUEUED, + ) + session.add(job) + session.commit() + job_id = job.id + session.close() + return job_id + + +def test_job_queue_claims_single_queued_job(session_factory): + job_id = _seed_job(session_factory) + + session = session_factory() + try: + queue = JobQueue(session) + claimed = queue.claim_next_job(worker_id="worker-1") + assert claimed is not None + assert claimed.id == job_id + assert claimed.status == JobStatus.RUNNING.value + finally: + session.close() + + +def test_job_queue_does_not_double_claim(session_factory): + _seed_job(session_factory) + + s1 = session_factory() + s2 = session_factory() + try: + q1 = JobQueue(s1) + q2 = JobQueue(s2) + first = q1.claim_next_job(worker_id="w-1") + second = q2.claim_next_job(worker_id="w-2") + assert first is not None + # Second worker must not see the already-claimed job (and no other jobs). + assert second is None + finally: + s1.close() + s2.close() + + +def test_job_queue_does_not_claim_other_tenants_job(session_factory): + _seed_job(session_factory, organization_id="org-a") + _seed_job(session_factory, organization_id="org-b") + + session = session_factory() + try: + queue = JobQueue(session) + # A worker processing org-a claims only org-a jobs. + claimed = queue.claim_next_job(worker_id="w-1", organization_id="org-a") + assert claimed is not None and claimed.organization_id == "org-a" + finally: + session.close() + + +def test_job_queue_complete_and_fail_update_status(session_factory): + job_id = _seed_job(session_factory) + + session = session_factory() + try: + queue = JobQueue(session) + claimed = queue.claim_next_job(worker_id="w-1") + assert claimed is not None + queue.complete_job(claimed.id, result={"ok": True}) + session.commit() + refreshed = session.get(Job, claimed.id) + assert refreshed.status == JobStatus.SUCCEEDED.value + assert refreshed.result == {"ok": True} + + job_id2 = _seed_job(session_factory) + claimed2 = queue.claim_next_job(worker_id="w-1") + assert claimed2 is not None and claimed2.id == job_id2 + queue.fail_job(claimed2.id, error_code="boom") + session.commit() + refreshed2 = session.get(Job, claimed2.id) + assert refreshed2.status == JobStatus.FAILED.value + assert refreshed2.error_code == "boom" + finally: + session.close() + + +def test_job_queue_none_when_empty(session_factory): + session = session_factory() + try: + queue = JobQueue(session) + assert queue.claim_next_job(worker_id="w-1") is None + finally: + session.close() + + +def test_job_queue_dispatch_invokes_registered_handler(session_factory): + calls = {} + + def handler(payload, job): + calls["payload"] = payload + calls["job_id"] = job.id + return {"processed": True} + + job_id = _seed_job(session_factory, operation="graph.build") + + session = session_factory() + try: + queue = JobQueue(session) + queue.register_handler("graph.build", handler) + claimed = queue.claim_next_job(worker_id="w-1") + assert claimed is not None + result = queue.dispatch(claimed, payload={"graph_id": "g1"}) + assert result == {"processed": True} + assert calls["job_id"] == job_id + assert calls["payload"] == {"graph_id": "g1"} + finally: + session.close() + + +def test_job_queue_dispatch_fails_unhandled_operation(session_factory): + job_id = _seed_job(session_factory, operation="unknown.op") + + session = session_factory() + try: + queue = JobQueue(session) + claimed = queue.claim_next_job(worker_id="w-1") + assert claimed is not None + with pytest.raises(ValueError, match="no_handler"): + queue.dispatch(claimed, payload={}) + # Claimed job is still running until the worker decides to fail it. + refreshed = session.get(Job, claimed.id) + assert refreshed.status == JobStatus.RUNNING.value + finally: + session.close() diff --git a/backend/tests/test_language_policy.py b/backend/tests/test_language_policy.py new file mode 100644 index 0000000..028bc29 --- /dev/null +++ b/backend/tests/test_language_policy.py @@ -0,0 +1,47 @@ +import importlib.util +from pathlib import Path +import sys +import unittest + + +_MODULE_PATH = Path(__file__).parents[1] / "app" / "utils" / "language_policy.py" +_SPEC = importlib.util.spec_from_file_location("language_policy_under_test", _MODULE_PATH) +assert _SPEC is not None and _SPEC.loader is not None +_LANGUAGE_POLICY = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _LANGUAGE_POLICY +_SPEC.loader.exec_module(_LANGUAGE_POLICY) + +DEFAULT_LOCALE = _LANGUAGE_POLICY.DEFAULT_LOCALE +SUPPORTED_LOCALES = _LANGUAGE_POLICY.SUPPORTED_LOCALES +locale_from_accept_language = _LANGUAGE_POLICY.locale_from_accept_language +normalize_locale = _LANGUAGE_POLICY.normalize_locale + + +class LanguagePolicyTests(unittest.TestCase): + def test_supported_locales_are_thai_and_english(self): + self.assertEqual(SUPPORTED_LOCALES, ("th", "en")) + self.assertEqual(DEFAULT_LOCALE, "th") + + def test_normalize_locale_accepts_language_region_values(self): + self.assertEqual(normalize_locale("en-US"), "en") + self.assertEqual(normalize_locale("TH_th"), "th") + + def test_normalize_locale_migrates_legacy_chinese_to_default(self): + self.assertEqual(normalize_locale("zh"), "th") + self.assertEqual(normalize_locale("zh-CN"), "th") + + def test_normalize_locale_falls_back_for_unknown_values(self): + self.assertEqual(normalize_locale(None), "th") + self.assertEqual(normalize_locale("fr"), "th") + self.assertEqual(normalize_locale(""), "th") + + def test_accept_language_uses_quality_and_skips_zero_quality(self): + header = "zh-CN;q=1, fr-FR;q=0.9, en-US;q=0.8, th;q=0" + self.assertEqual(locale_from_accept_language(header), "en") + + def test_accept_language_returns_default_when_no_supported_language_exists(self): + self.assertEqual(locale_from_accept_language("ja-JP, de;q=0.8"), "th") + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/tests/test_local_golden_flow.py b/backend/tests/test_local_golden_flow.py new file mode 100644 index 0000000..2bbca22 --- /dev/null +++ b/backend/tests/test_local_golden_flow.py @@ -0,0 +1,239 @@ +from types import SimpleNamespace +from typing import Any, cast + +from sqlalchemy import create_engine + +from app.config import Config +from app.db import Base, create_session_factory +from app.services.local_graph_builder import LocalGraphBuilderService +from app.services.memory_entity_reader import LocalEntityReader +from app.services.memory_extraction import ( + ExtractedEdge, + ExtractedEntity, + MemoryExtractionResult, +) +from app.services.memory_service import MemoryExtractionService +from app.services.memory_tools import LocalMemoryTools +from app.services.oasis_profile_generator import OasisProfileGenerator +from app.services.report_agent import ReportAgent +from app.services.report_agent import ReportManager, ReportOutline, ReportSection, ReportStatus +from app.services.simulation_config_generator import ( + AgentActivityConfig, + SimulationConfigGenerator, + SimulationParameters, +) +from app.services.simulation_manager import SimulationManager, SimulationStatus +from app.services.memory_entity_reader import make_local_entity_reader_factory + + +class DeterministicExtraction: + def extract(self, *, language, ontology, episode_text, context=""): + return MemoryExtractionResult( + entities=[ + ExtractedEntity( + mention="Alice", + canonical_name="Alice", + labels=["Entity", "Person"], + summary="A founder building Orbit.", + confidence=0.99, + ), + ExtractedEntity( + mention="Orbit", + canonical_name="Orbit", + labels=["Entity", "Organization"], + summary="A local project.", + confidence=0.99, + ), + ], + edges=[ + ExtractedEdge( + source_entity_ref="Alice", + target_entity_ref="Orbit", + relation="FOUNDED", + fact="Alice founded Orbit.", + confidence=0.98, + ) + ], + episode_summary="Alice founded Orbit.", + ) + + def persist(self, repository, result, *, source_type, source_ref, episode_text): + return MemoryExtractionService(cast(Any, None)).persist( + repository, + result, + source_type=source_type, + source_ref=source_ref, + episode_text=episode_text, + ) + + +def test_local_graph_profile_report_golden_flow(monkeypatch): + monkeypatch.setattr(Config, "MEMORY_BACKEND", "local") + monkeypatch.setattr(Config, "LLM_API_KEY", "test-key") + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + try: + builder = LocalGraphBuilderService( + session_factory, + organization_id="org-a", + project_id="project-a", + extraction_service=cast(Any, DeterministicExtraction()), + language="en", + ) + graph_id = builder.create_graph("golden") + builder.add_text_batches(graph_id, ["Alice founded Orbit."]) + + reader = LocalEntityReader( + session_factory(), + organization_id="org-a", + graph_id=graph_id, + owns_session=True, + ) + try: + filtered = reader.filter_defined_entities(defined_entity_types=["Person"]) + assert [entity.name for entity in filtered.entities] == ["Alice"] + + local_tools = LocalMemoryTools(reader.repository) + profile_generator = OasisProfileGenerator( + graph_id=graph_id, + use_zep_context=True, + local_memory_tools=local_tools, + ) + profile = profile_generator.generate_profile_from_entity( + cast(Any, filtered.entities[0]), + user_id=1, + use_llm=False, + ) + assert profile.name == "Alice" + assert profile.source_entity_uuid == filtered.entities[0].uuid + assert profile_generator.zep_client is None + + report_agent = ReportAgent( + graph_id=graph_id, + simulation_id="simulation-a", + simulation_requirement="Understand the project origin.", + llm_client=cast(Any, SimpleNamespace()), + memory_tools=local_tools, + ) + report_context = report_agent._execute_tool( + "quick_search", + {"query": "Alice", "limit": 10}, + ) + assert "Alice founded Orbit." in report_context + finally: + reader.close() + finally: + engine.dispose() + + +def test_local_graph_profile_simulation_report_is_persisted(monkeypatch, tmp_path): + monkeypatch.setattr(Config, "MEMORY_BACKEND", "local") + monkeypatch.setattr(Config, "LLM_API_KEY", "test-key") + monkeypatch.setattr(Config, "UPLOAD_FOLDER", str(tmp_path / "uploads")) + monkeypatch.setattr(ReportManager, "REPORTS_DIR", str(tmp_path / "reports")) + + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + simulation_dir = tmp_path / "simulations" + try: + builder = LocalGraphBuilderService( + session_factory, + organization_id="org-a", + project_id="project-a", + extraction_service=cast(Any, DeterministicExtraction()), + language="en", + ) + graph_id = builder.create_graph("e2e") + builder.add_text_batches(graph_id, ["Alice founded Orbit."]) + + monkeypatch.setattr( + SimulationConfigGenerator, + "generate_config", + lambda self, **kwargs: SimulationParameters( + simulation_id=kwargs["simulation_id"], + project_id=kwargs["project_id"], + graph_id=kwargs["graph_id"], + simulation_requirement=kwargs["simulation_requirement"], + agent_configs=[ + AgentActivityConfig( + agent_id=1, + entity_uuid="node-alice", + entity_name="Alice", + entity_type="Person", + ) + ], + generation_reasoning="deterministic-test", + ), + ) + + simulation_manager = SimulationManager( + entity_reader_factory=make_local_entity_reader_factory( + session_factory, + organization_id="org-a", + ) + ) + simulation_manager.SIMULATION_DATA_DIR = str(simulation_dir) + simulation = simulation_manager.create_simulation( + project_id="project-a", + graph_id=graph_id, + enable_twitter=False, + enable_reddit=True, + ) + prepared = simulation_manager.prepare_simulation( + simulation.simulation_id, + simulation_requirement="Understand the project origin.", + document_text="Alice founded Orbit.", + defined_entity_types=["Person"], + use_llm_for_profiles=False, + parallel_profile_count=1, + ) + assert prepared.status is SimulationStatus.READY + assert prepared.entities_count == 1 + assert prepared.profiles_count == 1 + assert simulation_manager.get_profiles(prepared.simulation_id) [0]["name"] == "Alice" + + report_session = session_factory() + try: + report_tools = LocalMemoryTools( + report_session, + organization_id="org-a", + graph_id=graph_id, + ) + report_agent = ReportAgent( + graph_id=graph_id, + simulation_id=prepared.simulation_id, + simulation_requirement="Understand the project origin.", + llm_client=cast(Any, SimpleNamespace()), + memory_tools=report_tools, + ) + monkeypatch.setattr( + ReportAgent, + "plan_outline", + lambda self, progress_callback=None: ReportOutline( + title="Local E2E Report", + summary="Evidence-backed local report.", + sections=[ReportSection(title="Evidence")], + ), + ) + monkeypatch.setattr( + ReportAgent, + "_generate_section_react", + lambda self, section, outline, previous_sections, progress_callback=None, section_index=0: self._execute_tool( + "quick_search", + {"query": "Alice", "limit": 10}, + ), + ) + + report = report_agent.generate_report(report_id="report-local-e2e") + assert report.status is ReportStatus.COMPLETED + assert "Alice founded Orbit." in report.markdown_content + persisted = ReportManager.get_report("report-local-e2e") + assert persisted is not None + assert persisted.status is ReportStatus.COMPLETED + assert "Alice founded Orbit." in persisted.markdown_content + finally: + report_session.close() + finally: + engine.dispose() diff --git a/backend/tests/test_local_graph_builder.py b/backend/tests/test_local_graph_builder.py new file mode 100644 index 0000000..4410837 --- /dev/null +++ b/backend/tests/test_local_graph_builder.py @@ -0,0 +1,134 @@ +from sqlalchemy import create_engine + +from app.db import Base, create_session_factory +from app.models.memory import MemoryGraph +from app.services.local_graph_builder import LocalGraphBuilderService + + +class FakeExtractionClient: + def __init__(self): + self.calls = [] + + def chat_json(self, messages, temperature=0.3, max_tokens=4096): + self.calls.append(messages) + return { + "entities": [ + { + "mention": "Alice", + "canonical_name": "Alice", + "labels": ["Person"], + "aliases": [], + "attributes": {"role": "founder"}, + "summary": "Alice founded the local project.", + "confidence": 0.95, + }, + { + "mention": "Orbit", + "canonical_name": "Orbit", + "labels": ["Project"], + "aliases": [], + "attributes": {}, + "summary": "Orbit is the project discussed in the episode.", + "confidence": 0.9, + }, + ], + "edges": [ + { + "source_entity_ref": "Alice", + "target_entity_ref": "Orbit", + "relation": "FOUNDED", + "fact": "Alice founded Orbit.", + "attributes": {}, + "valid_at": None, + "invalid_at": None, + "expired_at": None, + "confidence": 0.92, + "evidence": ["episode-0"], + } + ], + "episode_summary": "Alice founded Orbit.", + "unresolved_mentions": [], + } + + +def make_session_factory(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + return engine, create_session_factory(engine) + + +def test_local_graph_builder_persists_scoped_graph_and_extracted_memory(): + engine, session_factory = make_session_factory() + client = FakeExtractionClient() + builder = LocalGraphBuilderService( + session_factory, + organization_id="org-a", + project_id="project-a", + extraction_client=client, + ) + + graph_id = builder.create_graph(name="Orbit") + builder.set_ontology(graph_id, {"entity_types": ["Person", "Project"], "relations": ["FOUNDED"]}) + episode_ids = builder.add_text_batches( + graph_id, + ["Alice founded Orbit."], + batch_size=3, + ) + + assert len(episode_ids) == 1 + assert episode_ids[0].startswith("episode_") + assert client.calls + data = builder.get_graph_data(graph_id) + assert data["graph_id"] == graph_id + assert data["node_count"] == 2 + assert data["edge_count"] == 1 + + with session_factory() as session: + graph = session.get(MemoryGraph, graph_id) + assert graph.organization_id == "org-a" + assert graph.project_id == "project-a" + assert graph.ontology["relations"] == ["FOUNDED"] + + +def test_local_graph_builder_reprocessing_is_idempotent_for_episode_and_edge(): + engine, session_factory = make_session_factory() + builder = LocalGraphBuilderService( + session_factory, + organization_id="org-a", + project_id="project-a", + extraction_client=FakeExtractionClient(), + ) + graph_id = builder.create_graph(name="Orbit") + builder.set_ontology(graph_id, {"entity_types": ["Person", "Project"]}) + + builder.add_text_batches(graph_id, ["Alice founded Orbit."]) + builder.add_text_batches(graph_id, ["Alice founded Orbit."]) + + data = builder.get_graph_data(graph_id) + assert data["node_count"] == 2 + assert data["edge_count"] == 1 + + +def test_local_graph_builder_fails_closed_for_wrong_organization(): + engine, session_factory = make_session_factory() + builder = LocalGraphBuilderService( + session_factory, + organization_id="org-a", + project_id="project-a", + extraction_client=FakeExtractionClient(), + ) + graph_id = builder.create_graph(name="Orbit") + + other_builder = LocalGraphBuilderService( + session_factory, + organization_id="org-b", + project_id="project-b", + extraction_client=FakeExtractionClient(), + ) + + try: + other_builder.get_graph_data(graph_id) + except ValueError as exc: + assert str(exc) == "memory_graph_not_found" + else: + raise AssertionError("wrong organization must not read graph") diff --git a/backend/tests/test_local_graph_memory_updater.py b/backend/tests/test_local_graph_memory_updater.py new file mode 100644 index 0000000..d537d4c --- /dev/null +++ b/backend/tests/test_local_graph_memory_updater.py @@ -0,0 +1,50 @@ +from datetime import datetime, timezone + +from sqlalchemy import create_engine, select + +from app.db import Base, create_session_factory +from app.models.memory import MemoryEpisode, MemoryGraph +from app.services.local_graph_memory_updater import LocalGraphMemoryUpdater +from app.services.memory_activity import AgentActivity + + +def test_local_graph_memory_updater_persists_scoped_activity_episode(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + try: + with session_factory() as session: + session.add( + MemoryGraph( + id="graph-runtime", + organization_id="org-a", + project_id="project-a", + ) + ) + session.commit() + + updater = LocalGraphMemoryUpdater( + simulation_id="sim-runtime", + graph_id="graph-runtime", + organization_id="org-a", + session_factory=session_factory, + ) + updater.start() + updater.add_activity( + AgentActivity( + platform="twitter", + agent_id=7, + agent_name="Alice", + action_type="CREATE_POST", + action_args={"content": "Local memory is durable."}, + round_num=2, + timestamp=datetime.now(timezone.utc).isoformat(), + ) + ) + updater.stop() + + with session_factory() as session: + episodes = list(session.scalars(select(MemoryEpisode))) + assert len(episodes) == 1 + finally: + engine.dispose() diff --git a/backend/tests/test_local_reader_factory.py b/backend/tests/test_local_reader_factory.py new file mode 100644 index 0000000..f37b0df --- /dev/null +++ b/backend/tests/test_local_reader_factory.py @@ -0,0 +1,31 @@ +from sqlalchemy import create_engine + +from app.db import Base, create_session_factory +from app.models.memory import MemoryGraph +from app.services.memory_entity_reader import LocalEntityReader, make_local_entity_reader_factory + + +def test_local_reader_factory_scopes_each_reader_and_owns_worker_session(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + try: + with session_factory() as session: + session.add( + MemoryGraph( + id="graph-local", + organization_id="org-a", + project_id="project-a", + ) + ) + session.commit() + + factory = make_local_entity_reader_factory(session_factory, organization_id="org-a") + reader = factory("graph-local") + assert isinstance(reader, LocalEntityReader) + assert reader.repository.graph_id == "graph-local" + assert reader.repository.organization_id == "org-a" + assert reader.owns_session is True + reader.close() + finally: + engine.dispose() diff --git a/backend/tests/test_local_report_tools.py b/backend/tests/test_local_report_tools.py new file mode 100644 index 0000000..1016b11 --- /dev/null +++ b/backend/tests/test_local_report_tools.py @@ -0,0 +1,62 @@ +from types import SimpleNamespace + +from sqlalchemy import create_engine + +from app.db import Base, create_session_factory +from app.models.memory import MemoryEdge, MemoryGraph, MemoryNode +from app.services.memory_repository import SqlAlchemyMemoryRepository +from app.services.memory_tools import LocalMemoryTools +from app.services.report_agent import ReportAgent + + +def test_report_agent_tools_use_local_memory_contract_without_zep(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + with factory() as session: + session.add(MemoryGraph(id="graph-report", organization_id="org-a", project_id="project-a")) + session.flush() + repo = SqlAlchemyMemoryRepository(session, organization_id="org-a", graph_id="graph-report") + alice = repo.upsert_node( + canonical_name="Alice", + labels=["Entity", "Person"], + summary="Founder of Orbit", + ) + orbit = repo.upsert_node( + canonical_name="Orbit", + labels=["Entity", "Project"], + summary="A local project", + ) + repo.upsert_edge( + source_node_id=alice.id, + target_node_id=orbit.id, + relation="FOUNDED", + fact="Alice founded Orbit", + confidence=0.9, + ) + session.commit() + + tools = LocalMemoryTools(repo) + agent = ReportAgent( + graph_id="graph-report", + simulation_id="simulation-a", + simulation_requirement="Understand the project", + llm_client=SimpleNamespace(), + zep_tools=tools, + ) + + quick = agent._execute_tool("quick_search", {"query": "Alice"}) + insight = agent._execute_tool("insight_forge", {"query": "Who founded Orbit?"}) + panorama = agent._execute_tool("panorama_search", {"query": "Orbit"}) + stats = agent._execute_tool("get_graph_statistics", {}) + summary = agent._execute_tool("get_entity_summary", {"entity_name": "Alice"}) + by_type = agent._execute_tool("get_entities_by_type", {"entity_type": "Person"}) + + assert "Alice founded Orbit" in quick + assert "Alice founded Orbit" in insight + assert "Orbit" in panorama + assert '"node_count": 2' in stats + assert "Alice" in summary + assert "Alice" in by_type + + engine.dispose() diff --git a/backend/tests/test_local_service_import.py b/backend/tests/test_local_service_import.py new file mode 100644 index 0000000..a843430 --- /dev/null +++ b/backend/tests/test_local_service_import.py @@ -0,0 +1,168 @@ +import json +import os +import subprocess +import sys + +_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + + +def _py_env(): + env = os.environ.copy() + env["PYTHONPATH"] = _ROOT + return env + + +def _run_import_without_zep(module_name, symbol): + script = f""" +import builtins +original_import = builtins.__import__ + +def guarded_import(name, *args, **kwargs): + if name == 'zep_cloud' or name.startswith('zep_cloud.'): + raise RuntimeError('zep_imported_during_local_service_import') + return original_import(name, *args, **kwargs) + +builtins.__import__ = guarded_import +from app.services.{module_name} import {symbol} as imported_symbol +print(imported_symbol.__name__) +""" + return subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + env=_py_env(), + check=False, + ) + + +def _run_import_explicit_zep(module_name, symbol): + """Fresh import that REQUIRES zep_cloud, to prove explicit Zep paths still resolve.""" + script = f""" +import builtins +original_import = builtins.__import__ +_zep_seen = [] + +def tracking_import(name, *args, **kwargs): + if name == 'zep_cloud' or name.startswith('zep_cloud.'): + _zep_seen.append(name) + return original_import(name, *args, **kwargs) + +builtins.__import__ = tracking_import +from app.services.{module_name} import {symbol} as imported_symbol +print(imported_symbol.__name__) +print('ZEP_LOADED' if _zep_seen else 'NO_ZEP') +""" + return subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + env=_py_env(), + check=False, + ) + + +def test_local_service_import_does_not_eagerly_import_zep_client(): + result = _run_import_without_zep("local_graph_memory_updater", "LocalGraphMemoryUpdater") + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "LocalGraphMemoryUpdater" + + +def test_local_simulation_manager_import_does_not_eagerly_import_zep_client(): + result = _run_import_without_zep("simulation_manager", "SimulationManager") + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "SimulationManager" + + +def test_local_simulation_runner_import_does_not_eagerly_import_zep_client(): + result = _run_import_without_zep("simulation_runner", "SimulationRunner") + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "SimulationRunner" + + +def test_local_report_agent_import_does_not_eagerly_import_zep_client(): + result = _run_import_without_zep("report_agent", "ReportAgent") + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "ReportAgent" + + +def test_local_graph_builder_import_does_not_eagerly_import_zep_client(): + result = _run_import_without_zep("graph_builder", "GraphBuilderService") + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "GraphBuilderService" + + +def test_explicit_zep_tools_service_import_still_resolves(): + # The lazy refactor must not break explicit Zep-only consumers. + result = _run_import_explicit_zep("zep_tools", "ZepToolsService") + assert result.returncode == 0, result.stderr + lines = result.stdout.strip().splitlines() + assert lines[0] == "ZepToolsService" + assert "ZEP_LOADED" in lines + + +def test_explicit_zep_entity_reader_import_still_resolves(): + result = _run_import_explicit_zep("zep_entity_reader", "ZepEntityReader") + assert result.returncode == 0, result.stderr + lines = result.stdout.strip().splitlines() + assert lines[0] == "ZepEntityReader" + assert "ZEP_LOADED" in lines + + +_IDENTITY_PROBE = """ +import json +from app.services.memory_activity import AgentActivity as Shared +from app.services.zep_graph_memory_updater import ZepGraphMemoryUpdater + +zep_alias = getattr(ZepGraphMemoryUpdater, "AgentActivity", None) +if zep_alias is None: + import app.services.zep_graph_memory_updater as zup + zep_alias = getattr(zup, "AgentActivity", None) + +print(json.dumps( + { + "zep_alias_resolved": zep_alias is not None, + "shared_identity": zep_alias is Shared, + "module": Shared.__module__, + }, + ensure_ascii=False, +)) +""" + + +def test_local_and_zep_activity_contract_share_identity(): + result = subprocess.run( + [sys.executable, "-c", _IDENTITY_PROBE], + capture_output=True, + text=True, + env=_py_env(), + check=False, + ) + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout.strip()) + # When the Zep updater aliases the shared class, identity must be exact. + if payload["zep_alias_resolved"]: + assert payload["shared_identity"] is True + assert payload["module"] == "app.services.memory_activity" + + +_GRAPH_BUILDER_SEAM_PROBE = """ +from app.services import graph_builder +assert getattr(graph_builder, "Zep", None) is None +from app.services import GraphBuilderService as Cls +# The seam is a module-level sentinel that callers monkeypatch; it must remain +# reassignable without constructing a real Zep client at import time. +graph_builder.Zep = "sentinel" +print(getattr(graph_builder, "Zep", None)) +""" + + +def test_graph_builder_zep_seam_is_preserved_and_reassignable(): + result = subprocess.run( + [sys.executable, "-c", _GRAPH_BUILDER_SEAM_PROBE], + capture_output=True, + text=True, + env=_py_env(), + check=False, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "sentinel" diff --git a/backend/tests/test_local_zep_boundary.py b/backend/tests/test_local_zep_boundary.py new file mode 100644 index 0000000..04cc5d0 --- /dev/null +++ b/backend/tests/test_local_zep_boundary.py @@ -0,0 +1,34 @@ +import builtins + +from app.services.local_graph_memory_updater import LocalGraphMemoryUpdater + + +def test_local_activity_dict_path_does_not_import_zep_client(monkeypatch): + updater = object.__new__(LocalGraphMemoryUpdater) + captured = [] + setattr(updater, "add_activity", captured.append) + + original_import = builtins.__import__ + + def reject_zep_import(name, *args, **kwargs): + if name == "zep_cloud" or name.startswith("zep_cloud."): + raise AssertionError("local runtime imported the Zep client") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", reject_zep_import) + + updater.add_activity_from_dict( + { + "agent_id": 7, + "agent_name": "Alice", + "action_type": "CREATE_POST", + "action_args": {"content": "Local memory is durable."}, + "round": 2, + "timestamp": "2026-08-24T00:00:00+00:00", + }, + "twitter", + ) + + assert len(captured) == 1 + assert captured[0].__class__.__module__ == "app.services.memory_activity" + assert captured[0].to_episode_text().startswith("Alice:") diff --git a/backend/tests/test_locale_runtime.py b/backend/tests/test_locale_runtime.py new file mode 100644 index 0000000..c7edb79 --- /dev/null +++ b/backend/tests/test_locale_runtime.py @@ -0,0 +1,40 @@ +import unittest + +from flask import Flask + +from app.utils import locale as _LOCALE + + +_FLASK_APP = Flask(__name__) + + +class LocaleRuntimeTests(unittest.TestCase): + def tearDown(self): + _LOCALE.set_locale("th") + + def test_background_locale_normalizes_legacy_and_unknown_values(self): + _LOCALE.set_locale("zh-CN") + self.assertEqual(_LOCALE.get_locale(), "th") + _LOCALE.set_locale("en-US") + self.assertEqual(_LOCALE.get_locale(), "en") + _LOCALE.set_locale("fr") + self.assertEqual(_LOCALE.get_locale(), "th") + + def test_request_locale_negotiates_supported_language_only(self): + with _FLASK_APP.test_request_context( + headers={"Accept-Language": "zh-CN;q=1, en-US;q=0.8, th;q=0"} + ): + self.assertEqual(_LOCALE.get_locale(), "en") + + def test_translation_fallback_is_not_chinese(self): + _LOCALE.set_locale("zh") + translated = _LOCALE.t("api.projectNotFound", id="demo") + self.assertFalse(any("\u3400" <= char <= "\u9fff" for char in translated)) + self.assertNotEqual(translated, "api.projectNotFound") + self.assertFalse( + any("\u3400" <= char <= "\u9fff" for char in _LOCALE.get_language_instruction()) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/tests/test_memory_backend_switch.py b/backend/tests/test_memory_backend_switch.py new file mode 100644 index 0000000..2991773 --- /dev/null +++ b/backend/tests/test_memory_backend_switch.py @@ -0,0 +1,131 @@ +from types import SimpleNamespace + +import pytest +from flask import Flask, g +from sqlalchemy import create_engine + +from app.api import simulation_bp +from app.api.auth import auth_bp +from app.api.simulation import _simulation_manager_for_request +from app.config import Config +from app.db import Base, create_session_factory +from app.models.memory import MemoryGraph, MemoryNode +from app.models.project import ProjectManager +from app.security.policy import Role +from app.services.identity import IdentityRepository, PasswordService +from app.services.memory_entity_reader import LocalEntityReader +from app.utils.api_errors import ApiError + + +def make_app(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + app = Flask(__name__) + app.config.update(TESTING=True, SECRET_KEY="test-secret", SESSION_COOKIE_SECURE=False) + app.extensions["crowdsight_session_factory"] = create_session_factory(engine) + return app, engine + + +def test_local_memory_backend_selects_scoped_reader_factory(monkeypatch): + app, engine = make_app() + old_backend = Config.MEMORY_BACKEND + try: + monkeypatch.setattr(Config, "MEMORY_BACKEND", "local") + with app.test_request_context("/api/simulation/prepare"): + g.auth_context = SimpleNamespace( + user=SimpleNamespace(id="user-a"), + membership=SimpleNamespace(role=Role.USER), + organization=SimpleNamespace(id="org-a"), + ) + manager = _simulation_manager_for_request() + reader = manager.create_entity_reader("graph-a") + assert isinstance(reader, LocalEntityReader) + assert reader.repository.organization_id == "org-a" + reader.close() + finally: + Config.MEMORY_BACKEND = old_backend + engine.dispose() + + +def test_unknown_memory_backend_fails_closed(monkeypatch): + app, engine = make_app() + old_backend = Config.MEMORY_BACKEND + try: + monkeypatch.setattr(Config, "MEMORY_BACKEND", "unknown") + with app.test_request_context("/api/simulation/prepare"): + g.auth_context = SimpleNamespace( + user=SimpleNamespace(id="user-a"), + membership=SimpleNamespace(role=Role.USER), + organization=SimpleNamespace(id="org-a"), + ) + with pytest.raises(ApiError) as error: + _simulation_manager_for_request() + assert error.value.code == "invalid_memory_backend" + finally: + Config.MEMORY_BACKEND = old_backend + engine.dispose() + + +def test_local_entity_route_reads_scoped_memory_graph(monkeypatch): + app, engine = make_app() + app.register_blueprint(auth_bp, url_prefix="/api/auth") + app.register_blueprint(simulation_bp, url_prefix="/api/simulation") + original_projects_dir = ProjectManager.PROJECTS_DIR + old_backend = Config.MEMORY_BACKEND + old_zep_key = Config.ZEP_API_KEY + from tempfile import TemporaryDirectory + + with TemporaryDirectory() as temp_dir: + try: + ProjectManager.PROJECTS_DIR = f"{temp_dir}/projects" + with app.extensions["crowdsight_session_factory"]() as session: + repo = IdentityRepository(session) + org = repo.create_organization(name="Org A", slug="org-a") + user = repo.create_user( + email="user-a@example.com", + password_hash=PasswordService.hash_password("correct horse battery staple"), + ) + repo.create_membership(user.id, org.id, "user") + session.add( + MemoryGraph( + id="graph-local", + organization_id=org.id, + project_id="project-local", + ) + ) + session.add( + MemoryNode( + id="node-local", + graph_id="graph-local", + canonical_name="Local Alice", + normalized_name="local alice", + labels=["Person"], + summary="Stored locally.", + ) + ) + session.commit() + project = ProjectManager.create_project( + "Local project", + organization_id=org.id, + owner_user_id=user.id, + ) + project.graph_id = "graph-local" + ProjectManager.save_project(project) + + monkeypatch.setattr(Config, "MEMORY_BACKEND", "local") + monkeypatch.setattr(Config, "ZEP_API_KEY", None) + client = app.test_client() + login = client.post( + "/api/auth/login", + json={"email": "user-a@example.com", "password": "correct horse battery staple"}, + ) + assert login.status_code == 200 + response = client.get("/api/simulation/entities/graph-local") + assert response.status_code == 200 + entities = response.get_json()["data"]["entities"] + assert entities[0]["name"] == "Local Alice" + finally: + ProjectManager.PROJECTS_DIR = original_projects_dir + Config.MEMORY_BACKEND = old_backend + Config.ZEP_API_KEY = old_zep_key + engine.dispose() diff --git a/backend/tests/test_memory_entity_reader.py b/backend/tests/test_memory_entity_reader.py new file mode 100644 index 0000000..5ce26c8 --- /dev/null +++ b/backend/tests/test_memory_entity_reader.py @@ -0,0 +1,59 @@ +import json +from pathlib import Path + +from sqlalchemy import create_engine + +from app.db import Base, create_session_factory +from app.models.memory import MemoryEdge, MemoryGraph, MemoryNode +from app.services.memory_entity_reader import LocalEntityReader + + +FIXTURE = json.loads( + Path(__file__).with_name("fixtures").joinpath("memory_parity", "entity_reader_fixture.json").read_text() +) + + +def test_local_entity_reader_matches_entity_filter_fixture(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + try: + with create_session_factory(engine)() as session: + session.add( + MemoryGraph( + id=FIXTURE["graph_id"], + organization_id="org-a", + project_id="project-a", + ontology={"entity_types": FIXTURE["filter"]["defined_entity_types"]}, + ) + ) + session.add_all( + [ + MemoryNode(graph_id=FIXTURE["graph_id"], **node) + for node in FIXTURE["nodes"] + ] + ) + session.add_all( + [ + MemoryEdge(graph_id=FIXTURE["graph_id"], **edge) + for edge in FIXTURE["edges"] + ] + ) + session.commit() + + reader = LocalEntityReader(session, organization_id="org-a", graph_id=FIXTURE["graph_id"]) + result = reader.filter_defined_entities( + defined_entity_types=FIXTURE["filter"]["defined_entity_types"], + enrich_with_edges=True, + ) + assert result.total_count == FIXTURE["filter"]["total_count"] + assert result.filtered_count == FIXTURE["filter"]["filtered_count"] + assert sorted(result.entity_types) == FIXTURE["filter"]["entity_types"] + alice = next(entity for entity in result.entities if entity.name == "Alice") + assert alice.related_edges[0]["fact"] == "Alice works for Acme." + + detail = reader.get_entity_with_context("node-alice") + assert detail is not None + assert detail.get_entity_type() == "Person" + assert detail.related_nodes[0]["name"] == "Acme" + finally: + engine.dispose() diff --git a/backend/tests/test_memory_extraction.py b/backend/tests/test_memory_extraction.py new file mode 100644 index 0000000..9b7e8d4 --- /dev/null +++ b/backend/tests/test_memory_extraction.py @@ -0,0 +1,92 @@ +import json + +import pytest +from pydantic import ValidationError + +from app.services.memory_extraction import ( + MemoryExtractionResult, + build_extraction_prompt, + parse_extraction_response, +) + + +def test_parse_memory_extraction_result_with_entities_edges_and_evidence(): + raw = json.dumps( + { + "entities": [ + { + "mention": "Alice", + "canonical_name": "Alice", + "labels": ["Person"], + "aliases": ["A"], + "attributes": {"role": "founder"}, + "summary": "A founder.", + "confidence": 0.92, + } + ], + "edges": [ + { + "source_entity_ref": "Alice", + "target_entity_ref": "Bob", + "relation": "KNOWS", + "fact": "Alice knows Bob.", + "attributes": {}, + "valid_at": None, + "invalid_at": None, + "expired_at": None, + "confidence": 0.8, + "evidence": ["episode-1:0-15"], + } + ], + "episode_summary": "A relationship statement.", + "unresolved_mentions": ["Bob"], + } + ) + + result = parse_extraction_response(raw) + assert isinstance(result, MemoryExtractionResult) + assert result.entities[0].canonical_name == "Alice" + assert result.edges[0].relation == "KNOWS" + assert result.edges[0].evidence == ["episode-1:0-15"] + + +def test_parse_memory_extraction_result_rejects_unknown_fields_and_bad_confidence(): + payload = { + "entities": [], + "edges": [], + "episode_summary": "summary", + "unresolved_mentions": [], + "secret_prompt": "must not be accepted", + } + with pytest.raises(ValidationError): + parse_extraction_response(json.dumps(payload)) + + payload.pop("secret_prompt") + payload["entities"] = [ + { + "mention": "Alice", + "canonical_name": "Alice", + "labels": [], + "aliases": [], + "attributes": {}, + "summary": "", + "confidence": 1.5, + } + ] + with pytest.raises(ValidationError): + parse_extraction_response(payload) + + +def test_extraction_prompt_puts_language_instruction_first_and_bounds_episode_size(): + prompt = build_extraction_prompt( + language="th", + ontology={"entity_types": ["Person"], "edge_types": ["KNOWS"]}, + episode_text="Alice knows Bob.", + ) + assert prompt.startswith("IMPORTANT:") + assert "Return JSON only" in prompt + assert "entity_refs" in prompt + assert "Alice knows Bob." in prompt + + with pytest.raises(ValueError, match="episode_too_large"): + build_extraction_prompt(language="en", ontology={}, episode_text="x" * 20001) diff --git a/backend/tests/test_memory_parity.py b/backend/tests/test_memory_parity.py new file mode 100644 index 0000000..f5b61c6 --- /dev/null +++ b/backend/tests/test_memory_parity.py @@ -0,0 +1,154 @@ +import json +from datetime import datetime +from pathlib import Path + +import pytest +from sqlalchemy import create_engine + +from app.db import Base, create_session_factory +from app.models.memory import MemoryEdge, MemoryGraph, MemoryNode +from app.services.memory_entity_reader import LocalEntityReader +from app.services.memory_tools import LocalMemoryTools + + +FIXTURE = json.loads( + Path(__file__).with_name("fixtures").joinpath("memory_parity", "tools_fixture.json").read_text() +) +GRAPH_ID = FIXTURE["graph_id"] +ORG_ID = FIXTURE["organization_id"] + + +def _seed_fixture(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session = create_session_factory(engine)() + session.add( + MemoryGraph( + id=GRAPH_ID, + organization_id=ORG_ID, + project_id=FIXTURE["project_id"], + ) + ) + session.add_all([MemoryNode(graph_id=GRAPH_ID, **node) for node in FIXTURE["nodes"]]) + session.add_all( + [ + MemoryEdge( + graph_id=GRAPH_ID, + **{ + **edge, + "valid_at": _parse_datetime(edge["valid_at"]), + "invalid_at": _parse_datetime(edge["invalid_at"]), + "expired_at": _parse_datetime(edge["expired_at"]), + }, + ) + for edge in FIXTURE["edges"] + ] + ) + session.commit() + return engine, session + + +def _parse_datetime(value): + return datetime.fromisoformat(value) if value else None + + +def test_local_panorama_matches_legacy_all_graph_temporal_contract(): + engine, session = _seed_fixture() + try: + tools = LocalMemoryTools(session, organization_id=ORG_ID, graph_id=GRAPH_ID) + + result = tools.panorama_search(graph_id=GRAPH_ID, query="Alice", include_expired=True) + + assert {node.name for node in result.all_nodes} == {"Alice", "Acme", "Beta"} + assert {edge.fact for edge in result.all_edges} == { + "Alice works for Acme.", + "Alice previously worked for Acme.", + "Acme partnered with Beta.", + } + assert result.active_facts == ["Alice works for Acme.", "Acme partnered with Beta."] + assert len(result.historical_facts) == 1 + assert result.historical_facts[0].endswith("Alice previously worked for Acme.") + assert result.historical_facts[0].startswith("[2020-01-01T00:00:00") + finally: + session.close() + engine.dispose() + + +def test_local_panorama_keeps_graph_inventory_when_expired_facts_are_excluded(): + engine, session = _seed_fixture() + try: + tools = LocalMemoryTools(session, organization_id=ORG_ID, graph_id=GRAPH_ID) + + result = tools.panorama_search(graph_id=GRAPH_ID, query="not in any fact", include_expired=False) + + assert len(result.all_nodes) == 3 + assert len(result.all_edges) == 3 + assert result.active_facts == ["Alice works for Acme.", "Acme partnered with Beta."] + assert result.historical_facts == [] + finally: + session.close() + engine.dispose() + + +def test_local_insight_forge_matches_legacy_related_entity_contract(): + engine, session = _seed_fixture() + try: + tools = LocalMemoryTools(session, organization_id=ORG_ID, graph_id=GRAPH_ID) + + result = tools.insight_forge( + graph_id=GRAPH_ID, + query="Alice", + simulation_requirement="How might Alice's role change?", + ) + + assert result.sub_queries == ["Alice"] + assert result.semantic_facts == [ + "Alice works for Acme.", + "Alice previously worked for Acme.", + ] + assert {entity["uuid"] for entity in result.entity_insights} == {"node-alice", "node-acme"} + assert all("related_facts" in entity for entity in result.entity_insights) + assert all( + set(entity["related_facts"]) == set(result.semantic_facts) + for entity in result.entity_insights + ) + assert result.relationship_chains == [ + "Alice --[WORKS_FOR]--> Acme", + "Alice --[WORKED_FOR]--> Acme", + ] + assert result.total_facts == 2 + assert result.total_entities == 2 + assert result.total_relationships == 2 + finally: + session.close() + engine.dispose() + + +@pytest.mark.parametrize( + "operation", + [ + lambda reader: reader.get_all_nodes(graph_id="graph-other"), + lambda reader: reader.get_all_edges(graph_id="graph-other"), + lambda reader: reader.filter_defined_entities(graph_id="graph-other"), + lambda reader: reader.get_entity_with_context( + graph_id="graph-other", entity_uuid="node-alice" + ), + lambda reader: reader.get_entities_by_type( + graph_id="graph-other", entity_type="Person" + ), + ], +) +def test_local_entity_reader_rejects_conflicting_graph_scope(operation): + engine, session = _seed_fixture() + try: + reader = LocalEntityReader( + session, + organization_id=ORG_ID, + graph_id=GRAPH_ID, + ) + + with pytest.raises(ValueError, match="memory_graph_scope_conflict"): + operation(reader) + finally: + session.close() + engine.dispose() diff --git a/backend/tests/test_memory_repository.py b/backend/tests/test_memory_repository.py new file mode 100644 index 0000000..9c50c88 --- /dev/null +++ b/backend/tests/test_memory_repository.py @@ -0,0 +1,76 @@ +from sqlalchemy import create_engine + +from app.db import Base, create_session_factory +from app.models.memory import MemoryEdge, MemoryGraph, MemoryNode +from app.services.memory_repository import SqlAlchemyMemoryRepository + + +def make_memory_session(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + return engine, create_session_factory(engine) + + +def test_memory_repository_persists_graph_and_searches_structured_nodes(): + engine, session_factory = make_memory_session() + try: + with session_factory() as session: + repo = SqlAlchemyMemoryRepository(session, organization_id="org-a", graph_id="graph-a") + graph = repo.create_graph(project_id="project-a", ontology={"entity_types": ["Person"]}) + assert graph.id == "graph-a" + + alice = repo.upsert_node( + canonical_name="Alice", + labels=["Person"], + aliases=["A. Example"], + attributes={"role": "founder"}, + summary="A founder mentioned in the source.", + confidence=0.91, + ) + bob = repo.upsert_node(canonical_name="Bob", labels=["Person"]) + edge = repo.upsert_edge( + source_node_id=alice.id, + target_node_id=bob.id, + relation="KNOWS", + fact="Alice knows Bob.", + confidence=0.8, + ) + episode = repo.add_episode( + source_type="document", + source_ref="upload-1", + normalized_text="Alice knows Bob.", + summary="A relationship statement.", + ) + session.commit() + + assert edge.graph_id == graph.id + assert episode.graph_id == graph.id + results = repo.search("alice") + assert [node.canonical_name for node in results.nodes] == ["Alice"] + assert results.total_count == 1 + assert repo.get_node(alice.id).attributes == {"role": "founder"} + finally: + engine.dispose() + + +def test_memory_repository_fails_closed_for_wrong_organization_or_graph(): + engine, session_factory = make_memory_session() + try: + with session_factory() as session: + owner_repo = SqlAlchemyMemoryRepository(session, organization_id="org-a", graph_id="graph-a") + owner_repo.create_graph(project_id="project-a") + owner_node = owner_repo.upsert_node(canonical_name="Private Alice", labels=["Person"]) + other_repo = SqlAlchemyMemoryRepository(session, organization_id="org-b", graph_id="graph-b") + other_repo.create_graph(project_id="project-b") + other_node = other_repo.upsert_node(canonical_name="Private Bob", labels=["Person"]) + session.commit() + + wrong_org = SqlAlchemyMemoryRepository(session, organization_id="org-b", graph_id="graph-b") + assert wrong_org.get_node(owner_node.id) is None + assert wrong_org.search("alice").nodes == [] + + wrong_graph = SqlAlchemyMemoryRepository(session, organization_id="org-a", graph_id="graph-other") + assert wrong_graph.get_node(owner_node.id) is None + assert wrong_graph.get_node(other_node.id) is None + finally: + engine.dispose() diff --git a/backend/tests/test_memory_service.py b/backend/tests/test_memory_service.py new file mode 100644 index 0000000..7f9f86f --- /dev/null +++ b/backend/tests/test_memory_service.py @@ -0,0 +1,88 @@ +from app.services.memory_extraction import MemoryExtractionResult +from app.services.memory_repository import SqlAlchemyMemoryRepository +from app.services.memory_service import MemoryExtractionService +from app.db import Base, create_session_factory +from sqlalchemy import create_engine + + +class FakeLLM: + def __init__(self, payload): + self.payload = payload + self.messages = None + + def chat_json(self, messages, temperature=0.3, max_tokens=4096): + self.messages = messages + assert temperature <= 0.3 + assert max_tokens >= 4096 + return self.payload + + +def test_memory_extraction_service_calls_json_llm_and_persists_validated_result(): + payload = { + "entities": [ + { + "mention": "Alice", + "canonical_name": "Alice", + "labels": ["Person"], + "aliases": [], + "attributes": {}, + "summary": "Founder.", + "confidence": 0.9, + }, + { + "mention": "Bob", + "canonical_name": "Bob", + "labels": ["Person"], + "aliases": [], + "attributes": {}, + "summary": "Partner.", + "confidence": 0.8, + }, + ], + "edges": [ + { + "source_entity_ref": "Alice", + "target_entity_ref": "Bob", + "relation": "KNOWS", + "fact": "Alice knows Bob.", + "attributes": {}, + "valid_at": None, + "invalid_at": None, + "expired_at": None, + "confidence": 0.8, + "evidence": ["episode-1:0-15"], + } + ], + "episode_summary": "Relationship.", + "unresolved_mentions": [], + } + client = FakeLLM(payload) + service = MemoryExtractionService(client) + + result = service.extract( + language="en", + ontology={"entity_types": ["Person"], "edge_types": ["KNOWS"]}, + episode_text="Alice knows Bob.", + ) + assert isinstance(result, MemoryExtractionResult) + assert client.messages[0]["content"].startswith("IMPORTANT:") + + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + try: + with create_session_factory(engine)() as session: + repository = SqlAlchemyMemoryRepository(session, organization_id="org-a", graph_id="graph-a") + repository.create_graph(project_id="project-a") + ingest = service.persist( + repository, + result, + source_type="document", + source_ref="episode-1", + episode_text="Alice knows Bob.", + ) + session.commit() + assert ingest.entity_count == 2 + assert ingest.edge_count == 1 + assert repository.search("Alice").total_count == 1 + finally: + engine.dispose() diff --git a/backend/tests/test_memory_tools.py b/backend/tests/test_memory_tools.py new file mode 100644 index 0000000..65508cb --- /dev/null +++ b/backend/tests/test_memory_tools.py @@ -0,0 +1,39 @@ +import json +from pathlib import Path + +from sqlalchemy import create_engine + +from app.db import Base, create_session_factory +from app.models.memory import MemoryEdge, MemoryGraph, MemoryNode +from app.services.memory_tools import LocalMemoryTools + + +FIXTURE = json.loads( + Path(__file__).with_name("fixtures").joinpath("memory_parity", "entity_reader_fixture.json").read_text() +) + + +def test_local_memory_tools_quick_search_matches_golden_shape(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + try: + with create_session_factory(engine)() as session: + session.add(MemoryGraph(id="graph-fixture", organization_id="org-a", project_id="project-a")) + session.add_all([MemoryNode(graph_id="graph-fixture", **node) for node in FIXTURE["nodes"]]) + session.add_all([MemoryEdge(graph_id="graph-fixture", **edge) for edge in FIXTURE["edges"]]) + session.commit() + + tools = LocalMemoryTools(session, organization_id="org-a", graph_id="graph-fixture") + result = tools.quick_search("Alice", limit=10) + assert result.query == "Alice" + assert result.total_count == 1 + assert result.facts == ["Alice works for Acme."] + assert result.edges[0]["name"] == "WORKS_FOR" + assert result.nodes == [] + + both = tools.search_graph("Acme", limit=10, scope="both") + assert both.total_count == 2 + assert any(node["name"] == "Acme" for node in both.nodes) + assert any(edge["fact"] == "Alice works for Acme." for edge in both.edges) + finally: + engine.dispose() diff --git a/backend/tests/test_password_reset.py b/backend/tests/test_password_reset.py new file mode 100644 index 0000000..04fd881 --- /dev/null +++ b/backend/tests/test_password_reset.py @@ -0,0 +1,186 @@ +"""TDD gate: durable, single-use password reset tokens. + +Password resets must use a one-time, expiring, hashed token (never plaintext), +be tied to the user's auth_version so a reset invalidates old sessions, and be +scoped. The invite-pending password can only be finalized by the owner. +""" + +from datetime import timedelta + +import pytest +from sqlalchemy import create_engine + +from app.db import Base, create_session_factory +from app.services.password_reset import PasswordResetService + + +@pytest.fixture() +def session_factory(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + try: + yield factory + finally: + engine.dispose() + + +def test_password_reset_create_and_consume(session_factory): + session = session_factory() + try: + svc = PasswordResetService(session) + token = svc.create_token(user_id="user-1") + # Token is opaque and not the user id. + assert token and token != "user-1" + + # Consuming with the correct token succeeds and increments auth_version. + ok = svc.consume_token(token, user_id="user-1") + assert ok is True + + # Token is single-use: a second consume fails. + assert svc.consume_token(token, user_id="user-1") is False + finally: + session.close() + + +def test_password_reset_rejects_expired_token(session_factory): + session = session_factory() + try: + svc = PasswordResetService(session, ttl=timedelta(seconds=5)) + token = svc.create_token(user_id="user-1") + row = svc._latest_token(user_id="user-1") + # Force expiry. + row.expires_at = row.expires_at - timedelta(seconds=60) + session.commit() + assert svc.consume_token(token, user_id="user-1") is False + finally: + session.close() + + +def test_password_reset_rejects_wrong_user(session_factory): + session = session_factory() + try: + svc = PasswordResetService(session) + token = svc.create_token(user_id="user-1") + assert svc.consume_token(token, user_id="user-2") is False + finally: + session.close() + + +def test_password_reset_never_stores_plaintext_token(session_factory): + session = session_factory() + try: + svc = PasswordResetService(session) + token = svc.create_token(user_id="user-1") + row = svc._latest_token(user_id="user-1") + blob = repr(row) # hashed token in DB, plaintext token only returned to caller + assert token not in blob + finally: + session.close() + + +def test_password_reset_migration_round_trip(tmp_path, monkeypatch): + from pathlib import Path + + from alembic import command + from alembic.config import Config as AlembicConfig + from sqlalchemy import inspect + + from app.db import create_database_engine + + database_url = f"sqlite+pysqlite:///{tmp_path / 'pr-roundtrip.db'}" + monkeypatch.setenv("DATABASE_URL", database_url) + alembic_config = AlembicConfig(str(Path(__file__).resolve().parents[1] / "alembic.ini")) + alembic_config.set_main_option("sqlalchemy.url", database_url) + + command.upgrade(alembic_config, "head") + engine = create_database_engine(database_url) + try: + inspector = inspect(engine) + assert "password_reset_tokens" in set(inspector.get_table_names()) + cols = {c["name"] for c in inspector.get_columns("password_reset_tokens")} + assert {"user_id", "token_hash", "auth_version", "used", "expires_at"}.issubset(cols) + finally: + engine.dispose() + + command.downgrade(alembic_config, "0010_usage_events") + engine = create_database_engine(database_url) + try: + assert "password_reset_tokens" not in set(inspect(engine).get_table_names()) + finally: + engine.dispose() + + command.upgrade(alembic_config, "head") + command.check(alembic_config) + + +def test_password_reset_endpoints_full_flow(tmp_path, monkeypatch, session_factory): + """request → confirm flow works end to end and changes the password.""" + import secrets + + from flask import Flask, jsonify + + from app.api.auth import auth_bp + from app.services.identity import IdentityRepository, PasswordService + from app.services.password_reset import PasswordResetService + from app.utils.api_errors import ApiError + from app.utils.locale import t + + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + + app = Flask(__name__) + app.config.update(TESTING=True, SESSION_COOKIE_SECURE=False) + app.config["SECRET_KEY"] = secrets.token_hex(32) + app.extensions["crowdsight_session_factory"] = factory + + @app.errorhandler(ApiError) + def handle_api_error(error): + return jsonify(error.to_payload(t)), error.status_code + + app.register_blueprint(auth_bp, url_prefix="/api/auth") + + pr_email = "pr-flow@example.com" + with factory() as session: + repo = IdentityRepository(session) + org = repo.create_organization(name="PR Org", slug="pr-org") + user = repo.create_user( + email=pr_email, + password_hash=PasswordService.hash_password("old-password-12"), + ) + repo.create_membership(user.id, org.id, "user") + session.commit() + + client = app.test_client() + # Request (enrollment-agnostic). + r = client.post( + "/api/auth/password-reset/request", + json={"email": pr_email}, + ) + assert r.status_code == 200 + + # Issue a token directly against the store and confirm it. + with factory() as session: + user = IdentityRepository(session).get_user_by_email(pr_email) + token = PasswordResetService(session).create_token(user_id=user.id) + session.commit() + + r2 = client.post( + "/api/auth/password-reset/confirm", + json={"email": pr_email, "token": token, "password": "brand-new-pass-99"}, + ) + assert r2.status_code == 200 + + # Old password no longer works; new one does. + r_old = client.post( + "/api/auth/login", + json={"email": pr_email, "password": "old-password-12"}, + ) + assert r_old.status_code == 401 + r_new = client.post( + "/api/auth/login", + json={"email": pr_email, "password": "brand-new-pass-99"}, + ) + assert r_new.status_code == 200 + engine.dispose() diff --git a/backend/tests/test_product_persistence.py b/backend/tests/test_product_persistence.py new file mode 100644 index 0000000..fa63bfb --- /dev/null +++ b/backend/tests/test_product_persistence.py @@ -0,0 +1,781 @@ +"""TDD RED gate: durable product persistence for projects, simulations, and reports. + +The legacy ProjectManager/report/simulation managers persist to the filesystem. +This gate proves the durable SQL models + repository exist and enforce tenant/owner +scope, so product resources can migrate off filesystem state without losing +cross-tenant isolation. +""" + +from pathlib import Path + +import pytest +from alembic import command +from alembic.config import Config as AlembicConfig +from sqlalchemy import create_engine, inspect + +from app.db import Base, create_database_engine, create_session_factory +from app.models.product import ( + DurableReport, + ProductProject, + ProductSimulation, + ProjectStatus, + ReportStatus, + SimulationStatus, +) +from app.services.product_repository import ProductRepository + + +@pytest.fixture() +def session_factory(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + try: + yield factory + finally: + engine.dispose() + + +def test_product_project_persists_tenant_and_owner_scope(session_factory): + session = session_factory() + try: + project = ProductProject( + organization_id="org-a", + owner_user_id="user-1", + name="Q3 Brand Study", + status=ProjectStatus.CREATED, + language="en", + ) + session.add(project) + session.commit() + session.refresh(project) + + assert project.id + assert project.organization_id == "org-a" + assert project.owner_user_id == "user-1" + fetched = session.get(ProductProject, project.id) + assert fetched is not None + assert fetched.name == "Q3 Brand Study" + assert fetched.status == ProjectStatus.CREATED.value + finally: + session.close() + + +def test_product_project_lists_only_its_tenant(session_factory): + session = session_factory() + try: + session.add_all( + [ + ProductProject(organization_id="org-a", owner_user_id="user-1", name="A"), + ProductProject(organization_id="org-a", owner_user_id="user-2", name="B"), + ProductProject(organization_id="org-b", owner_user_id="user-1", name="C"), + ] + ) + session.commit() + + rows = ( + session.query(ProductProject) + .filter(ProductProject.organization_id == "org-a") + .all() + ) + assert {row.name for row in rows} == {"A", "B"} + finally: + session.close() + + +def test_product_simulation_scopes_to_project_and_creator(session_factory): + session = session_factory() + try: + project = ProductProject( + organization_id="org-a", owner_user_id="user-1", name="P" + ) + session.add(project) + session.flush() + + sim = ProductSimulation( + organization_id="org-a", + project_id=project.id, + created_by_user_id="user-1", + status=SimulationStatus.READY, + config={"graph_id": "graph-1"}, + ) + session.add(sim) + session.commit() + session.refresh(sim) + + assert sim.id + fetched = session.get(ProductSimulation, sim.id) + assert fetched.project_id == project.id + assert fetched.config["graph_id"] == "graph-1" + finally: + session.close() + + +def test_durable_report_scopes_to_project_simulation_and_creator(session_factory): + session = session_factory() + try: + project = ProductProject( + organization_id="org-a", owner_user_id="user-1", name="P" + ) + session.add(project) + session.flush() + sim = ProductSimulation( + organization_id="org-a", + project_id=project.id, + created_by_user_id="user-1", + status=SimulationStatus.READY, + config={}, + ) + session.add(sim) + session.flush() + + report = DurableReport( + organization_id="org-a", + project_id=project.id, + simulation_id=sim.id, + created_by_user_id="user-1", + status=ReportStatus.DRAFT, + title="Evidence Report", + ) + session.add(report) + session.commit() + session.refresh(report) + + fetched = session.get(DurableReport, report.id) + assert fetched is not None + assert fetched.title == "Evidence Report" + assert fetched.status == ReportStatus.DRAFT.value + finally: + session.close() + + +def test_product_project_rejects_conflicting_tenant_lookup(session_factory): + session = session_factory() + try: + project = ProductProject( + organization_id="org-a", owner_user_id="user-1", name="Secret" + ) + session.add(project) + session.commit() + + # A repository-level lookup must be tenant-scoped: an org-b caller cannot + # resolve an org-a project id. + row = ( + session.query(ProductProject) + .filter( + ProductProject.id == project.id, + ProductProject.organization_id == "org-b", + ) + .first() + ) + assert row is None + finally: + session.close() + + +def test_product_migrations_round_trip_and_match_models(tmp_path, monkeypatch): + database_url = f"sqlite+pysqlite:///{tmp_path / 'product-roundtrip.db'}" + monkeypatch.setenv("DATABASE_URL", database_url) + alembic_config = AlembicConfig(str(Path(__file__).resolve().parents[1] / "alembic.ini")) + alembic_config.set_main_option("sqlalchemy.url", database_url) + + command.upgrade(alembic_config, "head") + engine = create_database_engine(database_url) + try: + inspector = inspect(engine) + table_names = set(inspector.get_table_names()) + assert {"projects", "simulations", "reports"}.issubset(table_names) + + project_columns = {column["name"] for column in inspector.get_columns("projects")} + assert { + "organization_id", + "owner_user_id", + "name", + "status", + "language", + "ontology", + }.issubset(project_columns) + + project_fks = { + (tuple(fk["constrained_columns"]), tuple(fk["referred_columns"])) + for fk in inspector.get_foreign_keys("projects") + } + assert (("organization_id",), ("id",)) in project_fks + assert (("owner_user_id",), ("id",)) in project_fks + + simulation_columns = {column["name"] for column in inspector.get_columns("simulations")} + assert {"project_id", "created_by_user_id", "config", "status"}.issubset( + simulation_columns + ) + + report_columns = {column["name"] for column in inspector.get_columns("reports")} + assert { + "project_id", + "simulation_id", + "created_by_user_id", + "title", + "markdown_content", + }.issubset(report_columns) + finally: + engine.dispose() + + command.downgrade(alembic_config, "0006_job_metadata") + engine = create_database_engine(database_url) + try: + table_names = set(inspect(engine).get_table_names()) + assert "projects" not in table_names + assert "simulations" not in table_names + assert "reports" not in table_names + finally: + engine.dispose() + + command.upgrade(alembic_config, "head") + command.check(alembic_config) + + +def test_product_repository_scoped_project_crud(session_factory): + session = session_factory() + try: + repo = ProductRepository(session) + project = repo.create_project( + organization_id="org-a", owner_user_id="user-1", name="Repo Project" + ) + assert project.organization_id == "org-a" + assert project.name == "Repo Project" + + # Org-a can read it back by id. + fetched = repo.get_project(project.id, organization_id="org-a") + assert fetched is not None + assert fetched.name == "Repo Project" + + # Org-b cannot read an org-a project id (tenant isolation). + assert repo.get_project(project.id, organization_id="org-b") is None + + # Org-a owner list returns it. + listed = repo.list_projects(organization_id="org-a", owner_user_id="user-1") + assert [p.id for p in listed] == [project.id] + + # Another owner in the same org does not see it in their list. + assert repo.list_projects(organization_id="org-a", owner_user_id="user-2") == [] + finally: + session.close() + + +def test_product_repository_scoped_simulation_and_report(session_factory): + session = session_factory() + try: + repo = ProductRepository(session) + project = repo.create_project( + organization_id="org-a", owner_user_id="user-1", name="P" + ) + sim = repo.create_simulation( + organization_id="org-a", + project_id=project.id, + created_by_user_id="user-1", + config={"graph_id": "g1"}, + ) + assert repo.get_simulation(sim.id, organization_id="org-b") is None + assert repo.get_simulation(sim.id, organization_id="org-a") is not None + + report = repo.create_report( + organization_id="org-a", + project_id=project.id, + simulation_id=sim.id, + created_by_user_id="user-1", + title="R", + ) + assert repo.get_report(report.id, organization_id="org-b") is None + assert repo.get_report(report.id, organization_id="org-a") is not None + finally: + session.close() + + +def test_sync_project_state_to_durable(session_factory): + """Dual-write bridge: a legacy filesystem Project dict is mirrored to durable SQL.""" + from types import SimpleNamespace + + session = session_factory() + try: + repo = ProductRepository(session) + # Simulate the legacy ProjectManager.to_dict() output shape. + legacy_state = SimpleNamespace( + project_id="project-legacy", + name="Legacy Project", + organization_id="org-a", + owner_user_id="user-1", + status="created", + language="en", + total_text_length=120, + ontology={"entity_types": ["Person"]}, + simulation_requirement="Understand the market.", + graph_id=None, + graph_build_task_id=None, + error=None, + ) + durable = repo.sync_project(legacy_state, commit=True) + assert durable.id == "project-legacy" + assert durable.organization_id == "org-a" + assert durable.name == "Legacy Project" + assert durable.ontology == {"entity_types": ["Person"]} + + # It is now readable through the scoped repository. + fetched = repo.get_project("project-legacy", organization_id="org-a") + assert fetched is not None + # And tenant-isolated from org-b. + assert repo.get_project("project-legacy", organization_id="org-b") is None + finally: + session.close() + + +def test_sync_project_state_is_idempotent_on_repeat(session_factory): + from types import SimpleNamespace + + session = session_factory() + try: + repo = ProductRepository(session) + state = SimpleNamespace( + project_id="project-dup", + name="Project", + organization_id="org-a", + owner_user_id="user-1", + status="graph_completed", + language="en", + total_text_length=0, + ontology=None, + simulation_requirement=None, + graph_id="graph-9", + graph_build_task_id="job-9", + error=None, + ) + first = repo.sync_project(state, commit=True) + second = repo.sync_project(state, commit=True) + assert first.id == second.id + rows = ( + session.query(ProductProject) + .filter(ProductProject.id == "project-dup") + .all() + ) + assert len(rows) == 1 + assert rows[0].graph_id == "graph-9" + assert rows[0].status == "graph_completed" + finally: + session.close() + + +def test_api_sync_helper_writes_durable_copy_via_session_factory(session_factory, monkeypatch): + """The route helper mirrors a filesystem project into durable SQL.""" + + class FakeApp: + class Extensions(dict): + pass + + def __init__(self): + self.extensions = {"crowdsight_session_factory": session_factory} + + from types import SimpleNamespace + + from app.api import graph as graph_api + + class FakeCurrentApp: + __slots__ = ("_app",) + + def __init__(self, app): + object.__setattr__(self, "_app", app) + + @property + def extensions(self): + return object.__getattribute__(self, "_app").extensions + + fake = FakeCurrentApp(FakeApp()) + monkeypatch.setattr(graph_api, "current_app", fake) + + legacy = SimpleNamespace( + project_id="project-via-route", + name="Route Project", + organization_id="org-a", + owner_user_id="user-1", + status="created", + language="en", + total_text_length=0, + ontology=None, + simulation_requirement="Understand the market.", + graph_id=None, + graph_build_task_id=None, + error=None, + ) + graph_api._sync_project_to_durable(legacy) + + # Verify a durable row was created and is tenant-isolated. + session = session_factory() + try: + repo = ProductRepository(session) + fetched = repo.get_project("project-via-route", organization_id="org-a") + assert fetched is not None + assert fetched.name == "Route Project" + assert repo.get_project("project-via-route", organization_id="org-b") is None + finally: + session.close() + + +def test_api_sync_helper_is_safe_without_session_factory(session_factory, monkeypatch): + """Non-local backend (no session factory) must not raise.""" + + from app.api import graph as graph_api + + class FakeCurrentApp: + extensions = {} + + monkeypatch.setattr(graph_api, "current_app", FakeCurrentApp()) + graph_api._sync_project_to_durable(object()) # should not raise + + +def test_api_sync_helper_accepts_explicit_session_factory(session_factory, monkeypatch): + """Background-thread callers pass the captured session_factory explicitly.""" + + from types import SimpleNamespace + + from app.api import graph as graph_api + + class FakeCurrentApp: + extensions = {} + + # No current_app factory, so only the explicit argument can work. + monkeypatch.setattr(graph_api, "current_app", FakeCurrentApp()) + + legacy = SimpleNamespace( + project_id="project-thread", + name="Thread Project", + organization_id="org-a", + owner_user_id="user-1", + status="graph_completed", + language="en", + total_text_length=0, + ontology=None, + simulation_requirement=None, + graph_id="graph-thread-1", + graph_build_task_id=None, + error=None, + ) + graph_api._sync_project_to_durable(legacy, session_factory) + + session = session_factory() + try: + repo = ProductRepository(session) + fetched = repo.get_project("project-thread", organization_id="org-a") + assert fetched is not None + assert fetched.graph_id == "graph-thread-1" + finally: + session.close() + + +def test_sync_simulation_state_to_durable(session_factory): + """Legacy simulation state is mirrored to durable SQL and tenant-isolated.""" + from types import SimpleNamespace + + session = session_factory() + try: + repo = ProductRepository(session) + project = repo.create_project( + organization_id="org-a", owner_user_id="user-1", name="P" + ) + legacy = SimpleNamespace( + simulation_id="sim-legacy", + project_id=project.id, + graph_id="graph-1", + organization_id="org-a", + owner_user_id="user-1", + status="ready", + platform="parallel", + config={"graph_id": "graph-1"}, + current_round=2, + ) + durable = repo.sync_simulation(legacy, commit=True) + assert durable.id == "sim-legacy" + assert durable.project_id == project.id + + assert repo.get_simulation("sim-legacy", organization_id="org-a") is not None + assert repo.get_simulation("sim-legacy", organization_id="org-b") is None + finally: + session.close() + + +def test_api_sync_simulation_helper_writes_durable_copy(session_factory, monkeypatch): + """The simulation route helper mirrors a legacy state via session factory.""" + + from types import SimpleNamespace + + from app.services.simulation_manager import SimulationState, SimulationStatus + from app.api import simulation as sim_api + + class FakeCurrentApp: + extensions = {"crowdsight_session_factory": session_factory} + + monkeypatch.setattr(sim_api, "current_app", FakeCurrentApp()) + + session = session_factory() + try: + repo = ProductRepository(session) + project = repo.create_project( + organization_id="org-a", owner_user_id="user-1", name="P" + ) + state = SimulationState( + simulation_id="sim-via-route", + project_id=project.id, + graph_id="graph-1", + enable_twitter=False, + enable_reddit=True, + status=SimulationStatus.READY, + ) + sim_api._sync_simulation_to_durable( + state, + organization_id="org-a", + project_id=project.id, + created_by_user_id="user-1", + ) + fetched = repo.get_simulation("sim-via-route", organization_id="org-a") + assert fetched is not None + assert fetched.project_id == project.id + # A stranger tenant cannot read it. + assert repo.get_simulation("sim-via-route", organization_id="org-b") is None + finally: + session.close() + + +def test_sync_report_state_to_durable(session_factory): + """Legacy report state is mirrored to durable SQL and tenant-isolated.""" + from types import SimpleNamespace + + session = session_factory() + try: + repo = ProductRepository(session) + project = repo.create_project( + organization_id="org-a", owner_user_id="user-1", name="P" + ) + sim = repo.create_simulation( + organization_id="org-a", + project_id=project.id, + created_by_user_id="user-1", + config={}, + ) + legacy = SimpleNamespace( + report_id="report-legacy", + status="completed", + title="Local Report", + markdown_content="# Report\nEvidence.", + outline={"sections": ["Evidence"]}, + error=None, + ) + durable = repo.sync_report( + legacy, + organization_id="org-a", + project_id=project.id, + simulation_id=sim.id, + created_by_user_id="user-1", + commit=True, + ) + assert durable.id == "report-legacy" + assert durable.title == "Local Report" + assert "Evidence." in durable.markdown_content + + assert repo.get_report("report-legacy", organization_id="org-a") is not None + assert repo.get_report("report-legacy", organization_id="org-b") is None + finally: + session.close() + + +def test_api_sync_report_helper_writes_durable_copy(session_factory, monkeypatch): + """The report route helper mirrors a legacy report via session factory.""" + + from types import SimpleNamespace + + from app.api import report as report_api + + class FakeCurrentApp: + extensions = {"crowdsight_session_factory": session_factory} + + monkeypatch.setattr(report_api, "current_app", FakeCurrentApp()) + + session = session_factory() + try: + repo = ProductRepository(session) + project = repo.create_project( + organization_id="org-a", owner_user_id="user-1", name="P" + ) + sim = repo.create_simulation( + organization_id="org-a", + project_id=project.id, + created_by_user_id="user-1", + config={}, + ) + legacy = SimpleNamespace( + report_id="report-via-route", + status="completed", + title="Route Report", + markdown_content="Body.", + outline=None, + error=None, + ) + report_api._sync_report_to_durable( + legacy, + organization_id="org-a", + project_id=project.id, + simulation_id=sim.id, + created_by_user_id="user-1", + ) + fetched = repo.get_report("report-via-route", organization_id="org-a") + assert fetched is not None + assert fetched.title == "Route Report" + assert repo.get_report("report-via-route", organization_id="org-b") is None + finally: + session.close() + + +def test_scoped_project_reads_from_durable_first(session_factory, monkeypatch): + """Read cutover: scoped_project resolves from durable SQL, not filesystem.""" + + from types import SimpleNamespace + + from app.config import Config + from app.security import resources as res + + old_backend = Config.MEMORY_BACKEND + monkeypatch.setattr(Config, "MEMORY_BACKEND", "local") + session = session_factory() + try: + repo = ProductRepository(session) + durable = repo.create_project( + organization_id="org-a", owner_user_id="user-1", name="Durable Project" + ) + session.commit() + + class FakeCurrentApp: + extensions = {"crowdsight_session_factory": session_factory} + + monkeypatch.setattr(res, "current_app", FakeCurrentApp()) + monkeypatch.setattr( + res, + "current_actor", + lambda: SimpleNamespace( + organization_id="org-a", + user_id="user-1", + role="user", + ), + ) + + project = res.scoped_project(durable.id) + assert project is not None + assert project.project_id == durable.id + assert project.name == "Durable Project" + assert project.organization_id == "org-a" + assert project.owner_user_id == "user-1" + + # A cross-tenant caller cannot resolve it. + monkeypatch.setattr( + res, + "current_actor", + lambda: SimpleNamespace( + organization_id="org-b", + user_id="user-9", + role="user", + ), + ) + assert res.scoped_project(durable.id) is None + finally: + Config.MEMORY_BACKEND = old_backend + session.close() + + +def test_scoped_project_falls_back_to_filesystem_when_no_durable( + monkeypatch, tmp_path +): + """Read cutover: absent durable row falls back to legacy filesystem manager.""" + from types import SimpleNamespace + + from app.models.project import ProjectManager + from app.security import resources as res + + original_dir = ProjectManager.PROJECTS_DIR + ProjectManager.PROJECTS_DIR = str(tmp_path / "projects") + try: + fs_project = ProjectManager.create_project( + "FS Project", organization_id="org-a", owner_user_id="user-1" + ) + + class FakeCurrentApp: + extensions = {} + + monkeypatch.setattr(res, "current_app", FakeCurrentApp()) + monkeypatch.setattr( + res, + "current_actor", + lambda: SimpleNamespace( + organization_id="org-a", + user_id="user-1", + role="user", + ), + ) + project = res.scoped_project(fs_project.project_id) + assert project is not None + assert project.project_id == fs_project.project_id + finally: + ProjectManager.PROJECTS_DIR = original_dir + + +def test_simulation_state_save_mirrors_durable_when_scoped(session_factory, monkeypatch): + """Simulation status updates dual-write when state carries tenant scope.""" + from app.services.simulation_manager import ( + SimulationManager, + SimulationState, + SimulationStatus, + ) + + manager = SimulationManager(session_factory=session_factory) + state = SimulationState( + simulation_id="sim-status", + project_id="project-p", + graph_id="graph-g", + organization_id="org-a", + owner_user_id="user-1", + status=SimulationStatus.RUNNING, + current_round=3, + ) + # Trigger the same save path used during run; then confirm durable mirror. + manager._save_simulation_state(state) + + session = session_factory() + try: + repo = ProductRepository(session) + fetched = repo.get_simulation("sim-status", organization_id="org-a") + assert fetched is not None + assert fetched.status == "running" + assert fetched.current_round == 3 + assert fetched.organization_id == "org-a" + finally: + session.close() + + +def test_simulation_state_save_skips_durable_without_scope(session_factory): + """Unscoped legacy save must fall back to filesystem without raising.""" + from app.services.simulation_manager import ( + SimulationManager, + SimulationState, + SimulationStatus, + ) + + manager = SimulationManager(session_factory=session_factory) + state = SimulationState( + simulation_id="sim-noscope", + project_id="project-p", + graph_id="graph-g", + status=SimulationStatus.RUNNING, + ) + # No organization_id -> must not raise, and no durable row is written. + manager._save_simulation_state(state) + + session = session_factory() + try: + repo = ProductRepository(session) + assert repo.get_simulation("sim-noscope", organization_id="org-a") is None + finally: + session.close() + diff --git a/backend/tests/test_profile_memory_mode.py b/backend/tests/test_profile_memory_mode.py new file mode 100644 index 0000000..4c9cca3 --- /dev/null +++ b/backend/tests/test_profile_memory_mode.py @@ -0,0 +1,60 @@ +from types import SimpleNamespace + +from app.config import Config +from app.services.memory_entity_reader import LocalEntityNode +from app.services.oasis_profile_generator import OasisProfileGenerator + + +def test_profile_generator_can_disable_zep_context_for_local_memory(monkeypatch): + monkeypatch.setattr(Config, "LLM_API_KEY", "test-key") + monkeypatch.setattr(Config, "ZEP_API_KEY", "zep-key-that-must-not-be-used") + generator = OasisProfileGenerator( + graph_id="graph-local", + use_zep_context=False, + ) + assert generator.zep_api_key is None + assert generator.zep_client is None + + +def test_profile_generator_never_constructs_zep_in_local_backend(monkeypatch): + monkeypatch.setattr(Config, "MEMORY_BACKEND", "local") + monkeypatch.setattr(Config, "LLM_API_KEY", "test-key") + monkeypatch.setattr(Config, "ZEP_API_KEY", "zep-key-that-must-not-be-used") + + generator = OasisProfileGenerator(graph_id="graph-local") + + assert generator.zep_api_key is None + assert generator.zep_client is None + + +def test_profile_generator_uses_local_memory_context_without_zep(monkeypatch): + monkeypatch.setattr(Config, "LLM_API_KEY", "test-key") + + class FakeLocalMemoryTools: + def search_graph(self, query, *, limit=10, scope="both"): + assert query == "Alice" + assert limit == 30 + assert scope == "both" + return SimpleNamespace( + facts=["Alice founded Orbit"], + nodes=[{"name": "Orbit", "summary": "A local project"}], + ) + + generator = OasisProfileGenerator( + graph_id="graph-local", + use_zep_context=False, + local_memory_tools=FakeLocalMemoryTools(), + ) + entity = LocalEntityNode( + uuid="node-alice", + name="Alice", + labels=["Entity", "person"], + summary="A founder", + attributes={}, + ) + + context = generator._build_entity_context(entity) + + assert "Alice founded Orbit" in context + assert "Orbit" in context + assert generator.zep_client is None diff --git a/backend/tests/test_project_store_security.py b/backend/tests/test_project_store_security.py new file mode 100644 index 0000000..20149dd --- /dev/null +++ b/backend/tests/test_project_store_security.py @@ -0,0 +1,33 @@ +from pathlib import Path +from tempfile import TemporaryDirectory + +import pytest + +from app.models.project import ProjectManager + + +def test_project_paths_reject_traversal_and_absolute_components(): + with TemporaryDirectory() as temp_dir: + original = ProjectManager.PROJECTS_DIR + ProjectManager.PROJECTS_DIR = temp_dir + try: + with pytest.raises(ValueError, match="invalid_project_id"): + ProjectManager._get_project_dir("../escape") + with pytest.raises(ValueError, match="invalid_project_id"): + ProjectManager._get_project_dir("/tmp/escape") + assert ProjectManager.get_project("../escape") is None + finally: + ProjectManager.PROJECTS_DIR = original + + +def test_project_metadata_write_leaves_no_partial_temp_file(): + with TemporaryDirectory() as temp_dir: + original = ProjectManager.PROJECTS_DIR + ProjectManager.PROJECTS_DIR = temp_dir + try: + project = ProjectManager.create_project("atomic") + assert ProjectManager.get_project(project.project_id) is not None + project_files = {path.name for path in Path(temp_dir, project.project_id).iterdir()} + assert project_files == {"project.json", "files"} + finally: + ProjectManager.PROJECTS_DIR = original diff --git a/backend/tests/test_project_tenant_scope.py b/backend/tests/test_project_tenant_scope.py new file mode 100644 index 0000000..3289b84 --- /dev/null +++ b/backend/tests/test_project_tenant_scope.py @@ -0,0 +1,68 @@ +import tempfile +from pathlib import Path + +from app.models.project import ProjectManager + + +def test_project_manager_lists_only_requested_tenant_and_owner(): + original_dir = ProjectManager.PROJECTS_DIR + with tempfile.TemporaryDirectory() as temp_dir: + ProjectManager.PROJECTS_DIR = str(Path(temp_dir) / "projects") + try: + project_a = ProjectManager.create_project( + "A1", organization_id="org-a", owner_user_id="user-a" + ) + ProjectManager.create_project( + "A2", organization_id="org-a", owner_user_id="user-b" + ) + ProjectManager.create_project( + "B1", organization_id="org-b", owner_user_id="user-c" + ) + + org_a = ProjectManager.list_projects(organization_id="org-a") + assert {project.name for project in org_a} == {"A1", "A2"} + own = ProjectManager.list_projects( + organization_id="org-a", owner_user_id="user-a" + ) + assert [project.project_id for project in own] == [project_a.project_id] + finally: + ProjectManager.PROJECTS_DIR = original_dir + + +def test_project_scope_lookup_fails_closed_for_legacy_unowned_records(): + original_dir = ProjectManager.PROJECTS_DIR + with tempfile.TemporaryDirectory() as temp_dir: + ProjectManager.PROJECTS_DIR = str(Path(temp_dir) / "projects") + try: + legacy = ProjectManager.create_project("Legacy") + assert ProjectManager.get_project_for_scope( + legacy.project_id, organization_id="org-a", owner_user_id="user-a" + ) is None + finally: + ProjectManager.PROJECTS_DIR = original_dir + + +def test_delete_project_requires_matching_organization_and_owner_scope(): + original_dir = ProjectManager.PROJECTS_DIR + with tempfile.TemporaryDirectory() as temp_dir: + ProjectManager.PROJECTS_DIR = str(Path(temp_dir) / "projects") + try: + project = ProjectManager.create_project( + "Tenant project", organization_id="org-a", owner_user_id="user-a" + ) + + assert ProjectManager.delete_project( + project.project_id, + organization_id="org-b", + owner_user_id="user-b", + ) is False + assert ProjectManager.get_project(project.project_id) is not None + + assert ProjectManager.delete_project( + project.project_id, + organization_id="org-a", + owner_user_id="user-a", + ) is True + assert ProjectManager.get_project(project.project_id) is None + finally: + ProjectManager.PROJECTS_DIR = original_dir diff --git a/backend/tests/test_rate_limiter.py b/backend/tests/test_rate_limiter.py new file mode 100644 index 0000000..fe6cc3a --- /dev/null +++ b/backend/tests/test_rate_limiter.py @@ -0,0 +1,172 @@ +"""TDD gate: durable rate limiting. + +Rate limiting must be durable (survive restarts) and tenant-scoped so login / +admin mutation spikes cannot bypass limits. This gate proves a sliding-window +counter that rejects when a key exceeds the configured threshold. +""" + +from datetime import timedelta + +import pytest +from sqlalchemy import create_engine + +from app.db import Base, create_session_factory +from app.services.rate_limiter import RateLimiter + + +@pytest.fixture() +def session_factory(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + try: + yield factory + finally: + engine.dispose() + + +def test_rate_limiter_allows_under_threshold(session_factory): + session = session_factory() + try: + limiter = RateLimiter(session, window=timedelta(minutes=15), limit=5) + for _ in range(4): + assert limiter.check_and_record("login", key="user-a@example.com") is True + finally: + session.close() + + +def test_rate_limiter_rejects_over_threshold(session_factory): + session = session_factory() + try: + limiter = RateLimiter(session, window=timedelta(minutes=15), limit=3) + for _ in range(3): + assert limiter.check_and_record("login", key="user-b@example.com") is True + assert limiter.check_and_record("login", key="user-b@example.com") is False + finally: + session.close() + + +def test_rate_limiter_is_per_key(session_factory): + session = session_factory() + try: + limiter = RateLimiter(session, window=timedelta(minutes=15), limit=2) + assert limiter.check_and_record("login", key="a@example.com") is True + assert limiter.check_and_record("login", key="a@example.com") is True + assert limiter.check_and_record("login", key="a@example.com") is False + # A different key is unaffected. + assert limiter.check_and_record("login", key="b@example.com") is True + finally: + session.close() + + +def test_rate_limiter_prevents_reset_by_restart(session_factory): + # Two sessions/factories against the SAME engine prove durability. + from app.services.rate_limiter import RateLimitEvent + + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + try: + s1 = factory() + limiter1 = RateLimiter(s1, window=timedelta(minutes=15), limit=2) + assert limiter1.check_and_record("login", key="c@example.com") is True + assert limiter1.check_and_record("login", key="c@example.com") is True + assert limiter1.check_and_record("login", key="c@example.com") is False + s1.close() + + s2 = factory() + limiter2 = RateLimiter(s2, window=timedelta(minutes=15), limit=2) + assert limiter2.check_and_record("login", key="c@example.com") is False + s2.close() + finally: + engine.dispose() + + +def test_rate_limit_migration_round_trip(tmp_path, monkeypatch): + from pathlib import Path + + from alembic import command + from alembic.config import Config as AlembicConfig + from sqlalchemy import inspect + + from app.db import create_database_engine + + database_url = f"sqlite+pysqlite:///{tmp_path / 'rl-roundtrip.db'}" + monkeypatch.setenv("DATABASE_URL", database_url) + alembic_config = AlembicConfig(str(Path(__file__).resolve().parents[1] / "alembic.ini")) + alembic_config.set_main_option("sqlalchemy.url", database_url) + + command.upgrade(alembic_config, "head") + engine = create_database_engine(database_url) + try: + inspector = inspect(engine) + assert "rate_limit_events" in set(inspector.get_table_names()) + cols = {c["name"] for c in inspector.get_columns("rate_limit_events")} + assert {"operation", "key", "organization_id", "created_at"}.issubset(cols) + finally: + engine.dispose() + + command.downgrade(alembic_config, "0008_platform_settings") + engine = create_database_engine(database_url) + try: + assert "rate_limit_events" not in set(inspect(engine).get_table_names()) + finally: + engine.dispose() + + command.upgrade(alembic_config, "head") + command.check(alembic_config) + + +def test_login_endpoint_rate_limits_excessive_attempts(monkeypatch, tmp_path, session_factory): + """Login POST is rate-limited: after the threshold, returns 429.""" + import secrets + + from flask import Flask, jsonify + + from app.api.auth import auth_bp + from app.services.identity import IdentityRepository, PasswordService + from app.utils.api_errors import ApiError + from app.utils.locale import t + + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + + app = Flask(__name__) + app.config.update(TESTING=True, SESSION_COOKIE_SECURE=False) + app.config["SECRET_KEY"] = secrets.token_hex(32) + app.extensions["crowdsight_session_factory"] = factory + + @app.errorhandler(ApiError) + def handle_api_error(error): + return jsonify(error.to_payload(t)), error.status_code + + app.register_blueprint(auth_bp, url_prefix="/api/auth") + + rate_email = "rate-limit@example.com" + with factory() as session: + repo = IdentityRepository(session) + org = repo.create_organization(name="Rate Org", slug="rate-org") + user = repo.create_user( + email=rate_email, + password_hash=PasswordService.hash_password("correct-horse"), + ) + repo.create_membership(user.id, org.id, "user") + session.commit() + + client = app.test_client() + # Exceed the login threshold (5) with wrong password. + statuses = [] + for _ in range(5): + r = client.post( + "/api/auth/login", + json={"email": rate_email, "password": "wrong"}, + ) + statuses.append(r.status_code) + # The 6th attempt is rejected by the rate limiter before credential check. + r6 = client.post( + "/api/auth/login", + json={"email": rate_email, "password": "correct-horse"}, + ) + assert r6.status_code == 429 + engine.dispose() diff --git a/backend/tests/test_report_api_contract.py b/backend/tests/test_report_api_contract.py new file mode 100644 index 0000000..b51708f --- /dev/null +++ b/backend/tests/test_report_api_contract.py @@ -0,0 +1,44 @@ +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +from flask import Flask + +from app.api import report as report_api +from app.config import Config +from app.services.report_agent import ReportAgent + + +def test_report_agent_requires_explicit_memory_tools_in_local_mode(): + previous_backend = Config.MEMORY_BACKEND + Config.MEMORY_BACKEND = "local" + try: + with pytest.raises(ValueError, match="memory_tools_required_for_local_backend"): + ReportAgent( + graph_id="graph-a", + simulation_id="simulation-a", + simulation_requirement="test", + llm_client=SimpleNamespace(), + ) + finally: + Config.MEMORY_BACKEND = previous_backend + + +def test_report_api_internal_failure_uses_safe_error_envelope(): + app = Flask(__name__) + with app.app_context(), app.test_request_context("/api/report/report-secret"): + with patch.object( + report_api.ReportManager, + "get_report", + side_effect=RuntimeError("secret-path=/private/credential-token"), + ): + response, status_code = report_api.get_report("report-secret") + + body = response.get_json() + assert status_code == 500 + assert body["success"] is False + assert body["error_code"] == "internal_error" + assert "secret-path" not in str(body) + assert "Traceback" not in str(body) + assert "traceback" not in body + assert "error" not in body diff --git a/backend/tests/test_resource_auth_scope.py b/backend/tests/test_resource_auth_scope.py new file mode 100644 index 0000000..9000064 --- /dev/null +++ b/backend/tests/test_resource_auth_scope.py @@ -0,0 +1,174 @@ +from flask import Flask + +from app.api import report_bp, simulation_bp +from app.api.auth import auth_bp +from app.db import Base, create_session_factory +from app.models.project import ProjectManager +from app.models.task import TaskManager +from app.services.identity import IdentityRepository, PasswordService +from app.services.report_agent import Report, ReportManager, ReportStatus +from app.services.simulation_manager import SimulationManager + + +PASSWORD = "correct horse battery staple" + + +def make_resource_app(): + from sqlalchemy import create_engine + + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + app = Flask(__name__) + app.config.update(TESTING=True, SECRET_KEY="test-secret", SESSION_COOKIE_SECURE=False) + app.extensions["crowdsight_session_factory"] = session_factory + app.register_blueprint(auth_bp, url_prefix="/api/auth") + app.register_blueprint(simulation_bp, url_prefix="/api/simulation") + app.register_blueprint(report_bp, url_prefix="/api/report") + + with session_factory() as session: + repo = IdentityRepository(session) + organization = repo.create_organization(name="Org A", slug="org-a") + user = repo.create_user( + email="user-a@example.com", + password_hash=PasswordService.hash_password(PASSWORD), + ) + repo.create_membership(user.id, organization.id, "user") + user_b = repo.create_user( + email="user-b@example.com", + password_hash=PasswordService.hash_password(PASSWORD), + ) + repo.create_membership(user_b.id, organization.id, "user") + session.commit() + return app, engine, organization.id, user.id, user_b.id + + +def login(client): + response = client.post( + "/api/auth/login", + json={"email": "user-a@example.com", "password": PASSWORD}, + ) + assert response.status_code == 200 + + +def csrf_headers(client): + return {"X-CSRF-Token": client.get_cookie("crowdsight_csrf").value} + + +def test_simulation_and_report_routes_require_authentication(): + app, engine, _organization_id, _user_id, _user_b_id = make_resource_app() + try: + client = app.test_client() + protected_requests = [ + ("GET", "/api/simulation/list", None), + ("GET", "/api/simulation/sim_other", None), + ("POST", "/api/simulation/create", {"project_id": "project_other"}), + ("GET", "/api/report/list", None), + ("GET", "/api/report/report_other", None), + ("POST", "/api/report/generate", {"simulation_id": "sim_other"}), + ("POST", "/api/report/generate/status", {"task_id": "task_other"}), + ] + for method, path, payload in protected_requests: + response = client.open(path, method=method, json=payload) + assert response.status_code == 401, (method, path, response.status_code) + finally: + engine.dispose() + + +def test_authenticated_mutations_still_require_csrf(): + app, engine, _organization_id, _user_id, _user_b_id = make_resource_app() + try: + client = app.test_client() + login(client) + response = client.post( + "/api/simulation/create", + json={"project_id": "project_other"}, + ) + assert response.status_code == 403 + assert response.get_json()["error_code"] == "csrf_failed" + + response = client.post( + "/api/report/generate", + json={"simulation_id": "sim_other"}, + headers=csrf_headers(client), + ) + assert response.status_code in {404, 400} + finally: + engine.dispose() + + +def test_user_cannot_read_or_create_from_another_users_simulation_and_report(): + app, engine, organization_id, user_a_id, user_b_id = make_resource_app() + original_projects_dir = ProjectManager.PROJECTS_DIR + original_simulations_dir = SimulationManager.SIMULATION_DATA_DIR + original_reports_dir = ReportManager.REPORTS_DIR + from tempfile import TemporaryDirectory + + with TemporaryDirectory() as temp_dir: + try: + ProjectManager.PROJECTS_DIR = f"{temp_dir}/projects" + SimulationManager.SIMULATION_DATA_DIR = f"{temp_dir}/simulations" + ReportManager.REPORTS_DIR = f"{temp_dir}/reports" + + project_a = ProjectManager.create_project( + "A", organization_id=organization_id, owner_user_id=user_a_id + ) + project_a.graph_id = "graph-a" + ProjectManager.save_project(project_a) + project_b = ProjectManager.create_project( + "B", organization_id=organization_id, owner_user_id=user_b_id + ) + project_b.graph_id = "graph-b" + ProjectManager.save_project(project_b) + + manager = SimulationManager() + simulation_a = manager.create_simulation(project_a.project_id, "graph-a") + simulation_b = manager.create_simulation(project_b.project_id, "graph-b") + report_b = Report( + report_id="report-b", + simulation_id=simulation_b.simulation_id, + graph_id="graph-b", + simulation_requirement="private requirement", + status=ReportStatus.COMPLETED, + markdown_content="private report", + created_at="2026-08-23T00:00:00", + ) + ReportManager.save_report(report_b) + + client = app.test_client() + login(client) + assert client.get(f"/api/simulation/{simulation_b.simulation_id}").status_code == 404 + listed_simulations = client.get("/api/simulation/list").get_json()["data"] + assert all(item["simulation_id"] != simulation_b.simulation_id for item in listed_simulations) + assert client.get(f"/api/report/{report_b.report_id}").status_code == 404 + listed_reports = client.get("/api/report/list").get_json()["data"] + assert all(item["report_id"] != report_b.report_id for item in listed_reports) + + response = client.post( + "/api/simulation/create", + json={"project_id": project_b.project_id}, + headers=csrf_headers(client), + ) + assert response.status_code == 404 + + task_id_b = TaskManager().create_task( + "report_generate", + metadata={ + "organization_id": organization_id, + "owner_user_id": user_b_id, + "simulation_id": simulation_b.simulation_id, + }, + ) + + response = client.post( + "/api/report/generate/status", + json={"task_id": task_id_b}, + headers=csrf_headers(client), + ) + assert response.status_code == 404 + + finally: + ProjectManager.PROJECTS_DIR = original_projects_dir + SimulationManager.SIMULATION_DATA_DIR = original_simulations_dir + ReportManager.REPORTS_DIR = original_reports_dir + engine.dispose() diff --git a/backend/tests/test_saas_operations_schema.py b/backend/tests/test_saas_operations_schema.py new file mode 100644 index 0000000..ee58d60 --- /dev/null +++ b/backend/tests/test_saas_operations_schema.py @@ -0,0 +1,259 @@ +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest +from alembic import command +from alembic.config import Config +from sqlalchemy import inspect +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from app.db import Base, create_database_engine + +from app.models.operations import AuditLog, IdempotencyRecord, Job, JobStatus +from app.security.policy import Role +from app.services.idempotency import IdempotencyConflict, IdempotencyService +from app.services.identity import IdentityRepository + + +def make_session(): + engine = create_database_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + return engine, Session(engine) + + +def test_operations_migrations_round_trip_and_match_models(tmp_path, monkeypatch): + database_url = f"sqlite+pysqlite:///{tmp_path / 'migration-roundtrip.db'}" + monkeypatch.setenv("DATABASE_URL", database_url) + alembic_config = Config(str(Path(__file__).resolve().parents[1] / "alembic.ini")) + alembic_config.set_main_option("sqlalchemy.url", database_url) + + command.upgrade(alembic_config, "head") + engine = create_database_engine(database_url) + try: + inspector = inspect(engine) + assert {"jobs", "idempotency_records", "audit_logs"}.issubset( + set(inspector.get_table_names()) + ) + job_columns = {column["name"] for column in inspector.get_columns("jobs")} + assert {"message", "result", "progress_detail", "metadata"}.issubset(job_columns) + job_indexes = {index["name"] for index in inspector.get_indexes("jobs")} + assert {"ix_jobs_org_status_created", "ix_jobs_org_owner"}.issubset(job_indexes) + job_foreign_keys = { + (tuple(foreign_key["constrained_columns"]), tuple(foreign_key["referred_columns"])) + for foreign_key in inspector.get_foreign_keys("jobs") + } + assert (("organization_id",), ("id",)) in job_foreign_keys + assert (("owner_user_id",), ("id",)) in job_foreign_keys + finally: + engine.dispose() + + command.downgrade(alembic_config, "0003_memory") + engine = create_database_engine(database_url) + try: + assert "jobs" not in set(inspect(engine).get_table_names()) + finally: + engine.dispose() + + command.upgrade(alembic_config, "head") + command.check(alembic_config) + + +def test_sqlite_engine_enables_foreign_keys(): + engine = create_database_engine("sqlite+pysqlite:///:memory:") + try: + with engine.connect() as connection: + assert connection.exec_driver_sql("PRAGMA foreign_keys").scalar() == 1 + finally: + engine.dispose() + + +def test_operations_reject_unknown_organization_reference(): + engine, session = make_session() + try: + session.add( + Job( + organization_id="org_missing", + operation="graph.build", + status=JobStatus.QUEUED, + ) + ) + with pytest.raises(IntegrityError): + session.commit() + finally: + session.rollback() + session.close() + engine.dispose() + + +def test_operations_enforce_owner_fk_and_delete_actions(): + engine, session = make_session() + try: + repo = IdentityRepository(session) + org = repo.create_organization(name="FK Org", slug="fk-org") + user = repo.create_user(email="fk@example.com") + session.commit() + + session.add( + Job( + organization_id=org.id, + owner_user_id="usr_missing", + operation="graph.build", + status=JobStatus.QUEUED, + ) + ) + with pytest.raises(IntegrityError): + session.commit() + session.rollback() + + user_job = Job( + organization_id=org.id, + owner_user_id=user.id, + operation="graph.build", + status=JobStatus.QUEUED, + ) + org_job = Job( + organization_id=org.id, + operation="report.generate", + status=JobStatus.QUEUED, + ) + session.add_all([user_job, org_job]) + session.commit() + user_job_id = user_job.id + org_job_id = org_job.id + + session.delete(user) + session.commit() + persisted_user_job = session.get(Job, user_job_id) + assert persisted_user_job is not None + assert persisted_user_job.owner_user_id is None + + session.delete(org) + session.commit() + assert session.get(Job, org_job_id) is None + finally: + session.close() + engine.dispose() + + +def seed_membership(session): + repo = IdentityRepository(session) + org = repo.create_organization(name="Org A", slug="ops-org") + user = repo.create_user(email="ops@example.com") + membership = repo.create_membership(user.id, org.id, Role.ADMIN) + session.commit() + return org, user, membership + + +def test_operations_schema_has_durable_tables_and_tenant_indexes(): + engine, session = make_session() + try: + tables = set(inspect(engine).get_table_names()) + assert {"jobs", "idempotency_records", "audit_logs"}.issubset(tables) + assert any( + "organization_id" in index["column_names"] + for index in inspect(engine).get_indexes("jobs") + ) + finally: + session.close() + engine.dispose() + + +def test_job_and_audit_records_keep_org_and_owner_scope(): + engine, session = make_session() + try: + org, user, _membership = seed_membership(session) + job = Job( + organization_id=org.id, + owner_user_id=user.id, + operation="graph.build", + status=JobStatus.QUEUED, + ) + audit = AuditLog( + organization_id=org.id, + actor_user_id=user.id, + action="job.created", + target_type="job", + target_id=job.id, + details={"safe": True}, + ) + session.add_all([job, audit]) + session.commit() + assert job.organization_id == org.id + assert job.owner_user_id == user.id + assert audit.organization_id == org.id + assert audit.details == {"safe": True} + finally: + session.close() + engine.dispose() + + +def test_idempotency_reservation_replays_same_request_and_rejects_hash_conflict(): + engine, session = make_session() + try: + org, user, _membership = seed_membership(session) + service = IdempotencyService(session) + first = service.reserve( + organization_id=org.id, + user_id=user.id, + key="key-1", + request_body={"project_name": "A"}, + ) + session.commit() + assert first.is_new is True + + replay = service.reserve( + organization_id=org.id, + user_id=user.id, + key="key-1", + request_body={"project_name": "A"}, + ) + assert replay.is_new is False + assert replay.record.id == first.record.id + + try: + service.reserve( + organization_id=org.id, + user_id=user.id, + key="key-1", + request_body={"project_name": "B"}, + ) + except IdempotencyConflict: + pass + else: + raise AssertionError("same idempotency key accepted a different request") + finally: + session.close() + engine.dispose() + + +def test_idempotency_is_tenant_and_user_scoped(): + engine, session = make_session() + try: + repo = IdentityRepository(session) + org_a = repo.create_organization(name="A", slug="org-a-ops") + org_b = repo.create_organization(name="B", slug="org-b-ops") + user_a = repo.create_user(email="a-ops@example.com") + user_b = repo.create_user(email="b-ops@example.com") + repo.create_membership(user_a.id, org_a.id, Role.ADMIN) + repo.create_membership(user_b.id, org_b.id, Role.ADMIN) + session.commit() + service = IdempotencyService(session) + a = service.reserve( + organization_id=org_a.id, + user_id=user_a.id, + key="shared-key", + request_body={"x": 1}, + ) + b = service.reserve( + organization_id=org_b.id, + user_id=user_b.id, + key="shared-key", + request_body={"x": 1}, + ) + assert a.is_new is True + assert b.is_new is True + assert a.record.id != b.record.id + finally: + session.close() + engine.dispose() diff --git a/backend/tests/test_saas_schema.py b/backend/tests/test_saas_schema.py new file mode 100644 index 0000000..a21ce6e --- /dev/null +++ b/backend/tests/test_saas_schema.py @@ -0,0 +1,84 @@ +from sqlalchemy import create_engine, event, inspect +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from app.db import Base +from app.security.policy import Role +from app.services.identity import IdentityRepository, PasswordService + + +def make_session(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + return engine, Session(engine) + + +def seed_org_and_users(session): + repo = IdentityRepository(session) + org_a = repo.create_organization(name="Org A", slug="org-a") + org_b = repo.create_organization(name="Org B", slug="org-b") + user_a = repo.create_user(email="user-a@example.com") + repo.create_membership(user_a.id, org_a.id, Role.USER) + session.commit() + return repo, org_a, org_b, user_a + + +def test_schema_exposes_only_the_three_product_roles(): + assert {role.value for role in Role} == {"super_admin", "admin", "user"} + + +def test_repository_reads_are_tenant_scoped(): + engine, session = make_session() + try: + repo, org_a, org_b, user_a = seed_org_and_users(session) + assert repo.get_user_for_org(user_a.id, org_a.id) is not None + assert repo.get_user_for_org(user_a.id, org_b.id) is None + assert repo.list_users(org_b.id) == [] + finally: + session.close() + engine.dispose() + + +def test_membership_is_unique_per_user_and_organization(): + engine, session = make_session() + try: + repo, org_a, _org_b, user_a = seed_org_and_users(session) + try: + repo.create_membership(user_a.id, org_a.id, Role.USER) + session.commit() + except IntegrityError: + session.rollback() + else: + raise AssertionError("duplicate membership was accepted") + finally: + session.close() + engine.dispose() + + +def test_repository_does_not_commit_inside_mutations(): + engine, session = make_session() + commits = [] + + def on_before_commit(*_args): + commits.append(True) + + event.listen(Session, "before_commit", on_before_commit) + try: + repo = IdentityRepository(session) + repo.create_organization(name="Org A", slug="org-a") + repo.create_user(email="user-a@example.com") + assert commits == [] + session.commit() + assert len(commits) == 1 + finally: + event.remove(Session, "before_commit", on_before_commit) + session.close() + engine.dispose() + + +def test_password_service_hashes_and_verifies_without_plaintext_storage(): + password = "correct horse battery staple" + hashed = PasswordService.hash_password(password) + assert hashed != password + assert PasswordService.verify_password(hashed, password) is True + assert PasswordService.verify_password(hashed, "wrong password") is False diff --git a/backend/tests/test_settings_service.py b/backend/tests/test_settings_service.py new file mode 100644 index 0000000..7fcfc80 --- /dev/null +++ b/backend/tests/test_settings_service.py @@ -0,0 +1,117 @@ +"""TDD gate: versioned, redacted platform LLM settings. + +LLM settings are currently environment-only. This gate proves a durable, +versioned settings store that (a) keeps the API key encrypted (never plaintext +in the record or logs), (b) exposes only masked values to the API surface, and +(c) snapshots an effective settings version for reproducible jobs. +""" + +import pytest +from sqlalchemy import create_engine + +from app.config import Config +from app.db import Base, create_session_factory +from app.services.settings_service import SettingsService + + +@pytest.fixture() +def session_factory(monkeypatch): + monkeypatch.setattr(Config, "SECRET_KEY", "test-encryption-secret-key") + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + try: + yield factory + finally: + engine.dispose() + + +def test_settings_service_saves_and_reads_masked(session_factory): + session = session_factory() + try: + svc = SettingsService(session) + version = svc.save_settings( + {"model": "gpt-4o", "base_url": "https://api.example.com/v1"}, + api_key="sk-secret-123", + updated_by="user-1", + ) + assert version + + active = svc.active_settings() + assert active["settings"]["model"] == "gpt-4o" + # API key must be masked, never plaintext. + assert active["api_key"] != "sk-secret-123" + assert "sk-secret-123" not in repr(active) + + # Version metadata present. + assert active["version"] + assert active["updated_by"] == "user-1" + finally: + session.close() + + +def test_settings_service_never_stores_plaintext_api_key(session_factory): + session = session_factory() + try: + svc = SettingsService(session) + svc.save_settings({"model": "m"}, api_key="sk-plain-abc", updated_by="user-1") + row = svc._latest_row() + secret_ref = getattr(row, "secret_ref", None) or "" + blob = repr(row.settings) + " " + str(secret_ref) + assert "sk-plain-abc" not in blob + finally: + session.close() + + +def test_settings_service_snapshots_effective_settings_for_job(session_factory): + session = session_factory() + try: + svc = SettingsService(session) + version = svc.save_settings( + {"model": "gpt-4o"}, api_key="sk-secret-9", updated_by="user-1" + ) + snapshot = svc.snapshot_for_job() + assert snapshot["version"] == version + assert snapshot["settings"]["model"] == "gpt-4o" + # The job snapshot must NOT contain the plaintext api key. + assert "sk-secret-9" not in repr(snapshot) + # It carries a settings version, not the secret. + assert snapshot["settings_version"] == version + finally: + session.close() + + +def test_platform_settings_migration_round_trip(tmp_path, monkeypatch): + from pathlib import Path + + from alembic import command + from alembic.config import Config as AlembicConfig + from sqlalchemy import inspect + + from app.db import create_database_engine + + database_url = f"sqlite+pysqlite:///{tmp_path / 'settings-roundtrip.db'}" + monkeypatch.setenv("DATABASE_URL", database_url) + alembic_config = AlembicConfig(str(Path(__file__).resolve().parents[1] / "alembic.ini")) + alembic_config.set_main_option("sqlalchemy.url", database_url) + + command.upgrade(alembic_config, "head") + engine = create_database_engine(database_url) + try: + inspector = inspect(engine) + assert "platform_settings" in set(inspector.get_table_names()) + cols = {c["name"] for c in inspector.get_columns("platform_settings")} + assert {"version", "settings", "secret_ref", "active"}.issubset(cols) + finally: + engine.dispose() + + command.downgrade(alembic_config, "0007_product_resources") + engine = create_database_engine(database_url) + try: + assert "platform_settings" not in set(inspect(engine).get_table_names()) + finally: + engine.dispose() + + command.upgrade(alembic_config, "head") + command.check(alembic_config) + diff --git a/backend/tests/test_simulation_memory_injection.py b/backend/tests/test_simulation_memory_injection.py new file mode 100644 index 0000000..c68fec0 --- /dev/null +++ b/backend/tests/test_simulation_memory_injection.py @@ -0,0 +1,27 @@ +import pytest + +from app.config import Config +from app.services.simulation_manager import SimulationManager + + +def test_simulation_manager_uses_injected_entity_reader_factory(): + sentinel = object() + calls = [] + + def factory(graph_id): + calls.append(graph_id) + return sentinel + + manager = SimulationManager(entity_reader_factory=factory) + assert manager.create_entity_reader("graph-local") is sentinel + assert calls == ["graph-local"] + + +def test_local_backend_refuses_unscoped_zep_reader_fallback(monkeypatch, tmp_path): + monkeypatch.setattr(Config, "MEMORY_BACKEND", "local") + monkeypatch.setattr(SimulationManager, "SIMULATION_DATA_DIR", str(tmp_path)) + + manager = SimulationManager() + + with pytest.raises(ValueError, match="local_entity_reader_factory_required"): + manager.create_entity_reader("graph-local") diff --git a/backend/tests/test_usage_service.py b/backend/tests/test_usage_service.py new file mode 100644 index 0000000..ddde302 --- /dev/null +++ b/backend/tests/test_usage_service.py @@ -0,0 +1,120 @@ +"""TDD gate: durable LLM usage/cost accounting. + +Records per-organization, per-user LLM usage (model, input/output tokens, +estimated cost) so admin/super-admin accounting and quotas are possible. No +prompt content or secrets are stored. +""" + +import pytest +from sqlalchemy import create_engine + +from app.db import Base, create_session_factory +from app.services.usage_service import UsageService + + +@pytest.fixture() +def session_factory(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + try: + yield factory + finally: + engine.dispose() + + +def test_usage_service_records_event(session_factory): + session = session_factory() + try: + svc = UsageService(session) + entry_id = svc.record_event( + organization_id="org-a", + user_id="user-1", + operation="report.generate", + model="gpt-4o", + input_tokens=100, + output_tokens=50, + ) + assert entry_id + + total = svc.total_cost(organization_id="org-a") + assert total > 0 # deterministic small estimate; no prompt text stored + + rows = svc.list_events(organization_id="org-a", limit=10) + assert len(rows) == 1 + assert rows[0].model == "gpt-4o" + assert rows[0].input_tokens == 100 + finally: + session.close() + + +def test_usage_service_scopes_by_organization(session_factory): + session = session_factory() + try: + svc = UsageService(session) + svc.record_event( + organization_id="org-a", user_id="u1", operation="op", model="m", + input_tokens=1, output_tokens=1, + ) + svc.record_event( + organization_id="org-b", user_id="u2", operation="op", model="m", + input_tokens=1, output_tokens=1, + ) + rows_a = svc.list_events(organization_id="org-a", limit=10) + assert len(rows_a) == 1 + assert rows_a[0].organization_id == "org-a" + finally: + session.close() + + +def test_usage_event_never_stores_prompt_or_secret(session_factory): + session = session_factory() + try: + svc = UsageService(session) + svc.record_event( + organization_id="org-a", user_id="u1", operation="report.generate", + model="m", input_tokens=1, output_tokens=1, + ) + row = svc.list_events(organization_id="org-a", limit=1)[0] + blob = repr(row) + assert "prompt" not in blob.lower() # no raw prompt content persisted + finally: + session.close() + + +def test_usage_migration_round_trip(tmp_path, monkeypatch): + from pathlib import Path + + from alembic import command + from alembic.config import Config as AlembicConfig + from sqlalchemy import inspect + + from app.db import create_database_engine + + database_url = f"sqlite+pysqlite:///{tmp_path / 'usage-roundtrip.db'}" + monkeypatch.setenv("DATABASE_URL", database_url) + alembic_config = AlembicConfig(str(Path(__file__).resolve().parents[1] / "alembic.ini")) + alembic_config.set_main_option("sqlalchemy.url", database_url) + + command.upgrade(alembic_config, "head") + engine = create_database_engine(database_url) + try: + inspector = inspect(engine) + assert "usage_events" in set(inspector.get_table_names()) + cols = {c["name"] for c in inspector.get_columns("usage_events")} + assert { + "organization_id", "user_id", "operation", "model", + "input_tokens", "output_tokens", "estimated_cost", + }.issubset(cols) + finally: + engine.dispose() + + command.downgrade(alembic_config, "0009_rate_limit") + engine = create_database_engine(database_url) + try: + assert "usage_events" not in set(inspect(engine).get_table_names()) + finally: + engine.dispose() + + command.upgrade(alembic_config, "head") + command.check(alembic_config) diff --git a/backend/tests/test_worker.py b/backend/tests/test_worker.py new file mode 100644 index 0000000..f5c59d4 --- /dev/null +++ b/backend/tests/test_worker.py @@ -0,0 +1,86 @@ +"""TDD gate: durable worker loop run_once lifecycle. + +Proves a worker claims a job, invokes the registered handler, and records +success/failure on the durable job — the portable core of the production worker. +""" + +import pytest +from sqlalchemy import create_engine + +from app.db import Base, create_session_factory +from app.models.operations import Job, JobStatus +from worker import run_once + + +@pytest.fixture() +def session_factory(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + factory = create_session_factory(engine) + try: + yield factory + finally: + engine.dispose() + + +def _seed(session_factory, *, organization_id="org-a", operation="graph.build", result=None): + session = session_factory() + job = Job( + organization_id=organization_id, + owner_user_id="user-1", + operation=operation, + status=JobStatus.QUEUED, + result=result, + ) + session.add(job) + session.commit() + job_id = job.id + session.close() + return job_id + + +def test_run_once_processes_and_completes_job(session_factory): + job_id = _seed(session_factory, result={"graph_id": "g1"}) + + def handler(payload, job): + assert payload == {"graph_id": "g1"} + return {"ok": True} + + session = session_factory() + try: + handled = run_once( + session, + worker_id="w-1", + organization_id="org-a", + **{"graph.build": handler}, + ) + assert handled is True + session.expire_all() + refreshed = session.get(Job, job_id) + assert refreshed.status == JobStatus.SUCCEEDED.value + assert refreshed.result == {"ok": True} + finally: + session.close() + + +def test_run_once_returns_false_when_no_job(session_factory): + session = session_factory() + try: + assert run_once(session, worker_id="w-1") is False + finally: + session.close() + + +def test_run_once_marks_unhandled_job_failed(session_factory): + job_id = _seed(session_factory, operation="no.handler") + + session = session_factory() + try: + handled = run_once(session, worker_id="w-1", organization_id="org-a") + assert handled is True + session.expire_all() + refreshed = session.get(Job, job_id) + assert refreshed.status == JobStatus.FAILED.value + assert refreshed.error_code + finally: + session.close() diff --git a/backend/uv.lock b/backend/uv.lock index 5c94483..71c71cf 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -15,6 +15,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/8a/340a1555ae33d7354dbca4faa54948d76d89a27ceef032c8c3bc661d003e/aiofiles-25.1.0-py3-none-any.whl", hash = "sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695", size = 14668, upload-time = "2025-10-09T20:51:03.174Z" }, ] +[[package]] +name = "alembic" +version = "1.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/2b/e4153978368de59918115c9e01d3ebf58a558a7285efa7e960c383c4b59a/alembic-1.19.1.tar.gz", hash = "sha256:e0fca0518118c78acc493e31bcb5402f190057aaf6df8b5b95ce94c4789cf648", size = 2070816, upload-time = "2026-08-08T16:32:01.565Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/89/e62cc37b69ad357cc8ecd6e7367f5245f523d3cbb338a66197212bdf6749/alembic-1.19.1-py3-none-any.whl", hash = "sha256:b39018cb3d9413a19cbd54cf3c02ad33998641f0538eb77413a488a21c3e14be", size = 265946, upload-time = "2026-08-08T16:32:03.153Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -46,6 +60,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321, upload-time = "2024-02-06T09:43:09.663Z" }, ] +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi-bindings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/43/bb8b6e8708d49a5ab36781333af092d9f483b198a2710d01281204640055/argon2_cffi_bindings-26.1.0.tar.gz", hash = "sha256:63505c71542a44b68b1e38060450fb006404170da375feb31af153e7f9c6205d", size = 1790807, upload-time = "2026-08-20T07:44:22.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/d2/0ae991f1b2181e5be49007c574710a800ad36c2978683addb3e67c474e55/argon2_cffi_bindings-26.1.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:21ca0396fe5ec995dd54431c32698189666f9224810acfa752e50d2bd94d9df2", size = 25521, upload-time = "2026-08-20T07:32:43.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e4/ad91d8297638aa2258aad4501c306aca99480dfe76ccd638173fa3702db9/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78de2d65e0b9ea7ce9d1b1c3e87297b2d7305a02c266ee2a2d6910daddd7ee69", size = 27177, upload-time = "2026-08-20T07:32:44.158Z" }, + { url = "https://files.pythonhosted.org/packages/6f/86/5363df11b86d02cf3662208e7406496327649cc90eb365bf6f4e8a54a41f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:27f1821903e2ceadcb88ec2b45ef190897b7682449c772f4d9b53e42c520cf29", size = 26597, upload-time = "2026-08-20T07:32:45.172Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b5/a14dcc592652347dad23ee93b278a4da5d2a25c9ed3ebd10d68eea823a4f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d88e5f7e60f28ae0b0cc6b2f16c43e87cd642a196a86f85e0d8bb6fe016fc16d", size = 27403, upload-time = "2026-08-20T07:32:46.13Z" }, + { url = "https://files.pythonhosted.org/packages/b3/81/b4a20d4902af7f796390bf9245ff83c5217dfa7367efa1d14986956c482b/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:34b7d9c24a4165a2c61cc8ae11d44d48c9ce2830fb536cb7914e11fdd9962728", size = 27132, upload-time = "2026-08-20T07:32:47.13Z" }, + { url = "https://files.pythonhosted.org/packages/7e/1b/c8de358af07b1c490e0fcb863ef98e46ddb486e45567aca5a60bd68d9daa/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:224865cbbcb7a2bd1356741dff12b0134df726b6d44bb7b500df8e303cbd9e81", size = 27588, upload-time = "2026-08-20T07:32:48.087Z" }, + { url = "https://files.pythonhosted.org/packages/48/2f/7ee62a6e79f9309f9d9982d301b22a00010adb580c05c8109b94d7b33de0/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ffff613aaa9ce6236766e2fc6dc560bb5abde7a2e2416e3db1f9ae395a2b4dd4", size = 26785, upload-time = "2026-08-20T07:32:48.977Z" }, + { url = "https://files.pythonhosted.org/packages/e9/10/960d0ee93d4897741bcaf4799c697dae2d81499f66fd1ed042a7dd54c1f4/argon2_cffi_bindings-26.1.0-cp310-abi3-win32.whl", hash = "sha256:a86c069c91a747a2c4e5c51473590aeb48172fff9b2130d23729a42d98665ecb", size = 23898, upload-time = "2026-08-20T07:32:50.114Z" }, + { url = "https://files.pythonhosted.org/packages/6d/3a/0cc14a05810e6add9bce5e87693334baa2222de5f647fa31781885b6573f/argon2_cffi_bindings-26.1.0-cp310-abi3-win_amd64.whl", hash = "sha256:2c36ff87b5dfaa477d0bd51e9d7f6abdae7c8955d2983c97419085d842154b3e", size = 25730, upload-time = "2026-08-20T07:32:51.091Z" }, + { url = "https://files.pythonhosted.org/packages/4e/db/d83cf2af140547f0b9cdaece05b2dc2dcbf991be4667331d073eff771435/argon2_cffi_bindings-26.1.0-cp310-abi3-win_arm64.whl", hash = "sha256:f9c4420a7a864fe1b86ce35befc95b8e39fb852493b81cf798671ddc265de638", size = 24478, upload-time = "2026-08-20T07:32:52.111Z" }, + { url = "https://files.pythonhosted.org/packages/a0/b9/97f0370f99611b14efd384918613dd5cbda75f28d9bb1b677aacfeaa17df/argon2_cffi_bindings-26.1.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:19b562b1de4b9052ef1214a2821c44b6e6f22945daa102c32ae4eff929d8b6d8", size = 23055, upload-time = "2026-08-20T07:33:19.716Z" }, + { url = "https://files.pythonhosted.org/packages/ae/70/7eb3fe7bf00103cbbb569c51aef150661f22b734a782673a600ff0f52309/argon2_cffi_bindings-26.1.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49d525938467d52c923a890153c99087c9d5a937d1f6b585dbdba34ec82e397a", size = 24869, upload-time = "2026-08-20T07:33:20.671Z" }, + { url = "https://files.pythonhosted.org/packages/5b/4b/9d5919c6cb1f15df7406af0f99b048bd93936f112e3e8f4c8077bc2a9110/argon2_cffi_bindings-26.1.0-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1b0bcac4d490a237e18cf91f57352920c29f77f2fa39efd0813fb81298bf17ba", size = 24216, upload-time = "2026-08-20T07:33:21.653Z" }, + { url = "https://files.pythonhosted.org/packages/a3/34/32109943bace7729233cc4ee78530baa306d8cc3c6501a64ba8cb3b58129/argon2_cffi_bindings-26.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:0cc40f7b4050bb93eb67de95d2d759322fc7ce4930b9d645581ecf4913ec651e", size = 23584, upload-time = "2026-08-20T07:33:22.613Z" }, +] + [[package]] name = "astor" version = "0.8.1" @@ -321,16 +372,22 @@ name = "crowdsight-backend" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "alembic" }, + { name = "argon2-cffi" }, { name = "camel-ai" }, { name = "camel-oasis" }, { name = "chardet" }, { name = "charset-normalizer" }, + { name = "cryptography" }, { name = "flask" }, { name = "flask-cors" }, + { name = "gunicorn" }, { name = "openai" }, + { name = "psycopg", extra = ["binary"] }, { name = "pydantic" }, { name = "pymupdf" }, { name = "python-dotenv" }, + { name = "sqlalchemy" }, { name = "zep-cloud" }, ] @@ -349,19 +406,25 @@ dev = [ [package.metadata] requires-dist = [ + { name = "alembic", specifier = ">=1.13,<2" }, + { name = "argon2-cffi", specifier = ">=23.1" }, { name = "camel-ai", specifier = "==0.2.78" }, { name = "camel-oasis", specifier = "==0.2.5" }, { name = "chardet", specifier = ">=5.0.0" }, { name = "charset-normalizer", specifier = ">=3.0.0" }, + { name = "cryptography", specifier = ">=41.0" }, { name = "flask", specifier = ">=3.0.0" }, { name = "flask-cors", specifier = ">=6.0.0" }, + { name = "gunicorn", specifier = ">=21.2.0" }, { name = "openai", specifier = ">=1.0.0" }, { name = "pipreqs", marker = "extra == 'dev'", specifier = ">=0.5.0" }, + { name = "psycopg", extras = ["binary"], specifier = ">=3.2" }, { name = "pydantic", specifier = ">=2.0.0" }, { name = "pymupdf", specifier = ">=1.24.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, + { name = "sqlalchemy", specifier = ">=2.0,<3" }, { name = "zep-cloud", specifier = "==3.13.0" }, ] provides-extras = ["dev"] @@ -567,6 +630,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/c7/b64cae5dba3a1b138d7123ec36bb5ccd39d39939f18454407e5468f4763f/fsspec-2025.12.0-py3-none-any.whl", hash = "sha256:8bf1fe301b7d8acfa6e8571e3b1c3d158f909666642431cc78a1b7b4dbc5ec5b", size = 201422, upload-time = "2025-12-03T15:23:41.434Z" }, ] +[[package]] +name = "greenlet" +version = "3.5.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/d8/7cc97c142388aef03f622e001c572c4f84e9252a439549d483f555771970/greenlet-3.5.5.tar.gz", hash = "sha256:adb4bae02e91a8e863e48b177e4014bdcac8a6b5e047ea1df687a61534b85e6c", size = 207585, upload-time = "2026-08-10T15:09:36.136Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/a3/07297917485ee2ca85bc3c8dc6ed85ad3fffcf424047fba62671dba68e97/greenlet-3.5.5-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:be63afcbbccfad3dd95a1ba12ada84dab2ef32031973d80b5b92df67fa763a61", size = 294165, upload-time = "2026-08-10T13:25:17.987Z" }, + { url = "https://files.pythonhosted.org/packages/db/51/6f732f9314cda54c5fd48a7620c7160f4f286967e8045ad94b9d66ce80b7/greenlet-3.5.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a268024ce2d7d2b04694bf1594058981a9fa663d1df4b762dee499211ed7c1c", size = 613610, upload-time = "2026-08-10T14:14:33.829Z" }, + { url = "https://files.pythonhosted.org/packages/d8/c0/b27589e25d220289edcd4d582b2b17b83058d1a56d53d971b6ea1a34f10d/greenlet-3.5.5-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:35cbb8bf55ace57fbccb4fb8622c4521713acd8691e77f4696d416ea7ca527da", size = 625481, upload-time = "2026-08-10T14:27:23.647Z" }, + { url = "https://files.pythonhosted.org/packages/51/2d/f2c928218ac52f26d7a2c188c171d1b7e728b23782cb3347e7b4fce1493a/greenlet-3.5.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74cc6df89ec5302337adc9cf096221cbed2510fd444b0e0f1586cf0470740864", size = 624562, upload-time = "2026-08-10T13:40:48.064Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4a/92fc51d5d35912f4f06eec037ba347985defd0be47463a010a325634d9d2/greenlet-3.5.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d9b454c5fc48aeaa7c4337813dbf513a6870468e426438a04d922c6d0fe63db", size = 1584909, upload-time = "2026-08-10T14:15:04.343Z" }, + { url = "https://files.pythonhosted.org/packages/ac/58/ed98b80ac5738c149a5258544843c45601ade1fd70f61740cdaead6351b3/greenlet-3.5.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03551ed792cb1b4fc0277a0c60dfd8c343894a0ba06fe60dcd22f568b433da39", size = 1651184, upload-time = "2026-08-10T13:40:28.879Z" }, + { url = "https://files.pythonhosted.org/packages/d8/be/b582ceb80cefdf9d8da34078714e4b12b3d16f509dee0f65e40a5cc8fc7d/greenlet-3.5.5-cp311-cp311-win_amd64.whl", hash = "sha256:ab3df3dffb58bf70564e93a5cec7941e4d9faa5a36cc4234a10d3131afe04f53", size = 323280, upload-time = "2026-08-10T13:26:07.495Z" }, + { url = "https://files.pythonhosted.org/packages/4d/18/5313c4c58598c38b0373c013e4ff2b3e6d258aaaa338f373335ebecdaddd/greenlet-3.5.5-cp311-cp311-win_arm64.whl", hash = "sha256:2b70a766135540c472ac1393d57c2e1b4a2eb85bf526a1e41e6d096173a8cee5", size = 307785, upload-time = "2026-08-10T13:28:34.874Z" }, + { url = "https://files.pythonhosted.org/packages/2e/7e/9ecd0285e3153532ae07aeb88063c43c72b4221cf0d4d123b02f3682e3ff/greenlet-3.5.5-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:49520f0c95a48b42cf55414b8e8479beb274ea70431afc33e3f79903c71f4380", size = 295809, upload-time = "2026-08-10T13:25:34.023Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/60e4bbcc89252037b18087f2ec16405d5b2d5be42dde191bbf3667e96102/greenlet-3.5.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55272212cbc5f43d1d723725ab931f1939969b7e9523882ca58b55061769d053", size = 611910, upload-time = "2026-08-10T14:14:35.18Z" }, + { url = "https://files.pythonhosted.org/packages/a4/17/cd5134be659cd4a443e7a61ae670dabec165a814c51162916d637b6dd38e/greenlet-3.5.5-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:655bca754a2ef4efcb0eb48a94d3f4593536d0f3d48f8ed44343c01d16a92f95", size = 624198, upload-time = "2026-08-10T14:27:25.229Z" }, + { url = "https://files.pythonhosted.org/packages/78/ac/5c5b959999b6f09c3026b5dfe171575bc3121c5236ce74f495096f25b203/greenlet-3.5.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:147b25a42e5ca5be3d42356e8f608b37af715a1c196e9bf9d1627f3341adfe1d", size = 621439, upload-time = "2026-08-10T13:40:49.391Z" }, + { url = "https://files.pythonhosted.org/packages/c8/8b/6acf112ed8aee499f25b4d6949820fb02ac950ff9c1f3d793bd5be0599f2/greenlet-3.5.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:27493374cff1d1b7919dc8126547f2aea582737e3046147b434b1e12de56389b", size = 1581342, upload-time = "2026-08-10T14:15:05.653Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d7/734e5f198888876b42d7616ff6644c075baf6b8a2412deadd6b0e1b8b20c/greenlet-3.5.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:12e2ee66c2aba86133f10fd99d6a8856c6d351ffb7be0e4d52ef2cc5fbb705b2", size = 1645744, upload-time = "2026-08-10T13:40:30.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/30/1f42b88dc587b5899ee50616ad56ee40cafaf225df4fb829f10183c62a5c/greenlet-3.5.5-cp312-cp312-win_amd64.whl", hash = "sha256:49ddacd36af37735fab103846f4ee4d18a492dde72730d1699c0c8ebe30d9f18", size = 324171, upload-time = "2026-08-10T13:28:44.472Z" }, + { url = "https://files.pythonhosted.org/packages/76/e5/4dee4d8d2e603fe5fdd7b444e63219f7b9bd852c60c6214511c7157cbe88/greenlet-3.5.5-cp312-cp312-win_arm64.whl", hash = "sha256:5f1b1ff4828cdc1aba4266aff814085d04a1d07959287219af021b838b265d52", size = 308362, upload-time = "2026-08-10T13:26:46.839Z" }, +] + +[[package]] +name = "gunicorn" +version = "26.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/8a/e4ef6ee11701b6cd64702848415ffb69eeff85cb388a3c6c7fe86f22f3f8/gunicorn-26.2.0.tar.gz", hash = "sha256:62b864895d9ebff0b2f9867ba04fe811c93121596540830c9c916d0769668447", size = 787921, upload-time = "2026-08-24T15:05:59.3Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/85/7522a52e5e2f42faf1a129113ab63e548c42e103e9af395b7bfe65e403e2/gunicorn-26.2.0-py3-none-any.whl", hash = "sha256:bd249d0b3f7972f7432f0a6b6ff3b3ee2d129f70cd1ff6c09a9dd9e29a2b88e3", size = 228389, upload-time = "2026-08-24T15:05:57.67Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -960,6 +1056,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/77/d7f491cbc05303ac6801651aabeb262d43f319288c1ea96c66b1d2692ff3/lxml-6.0.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:27220da5be049e936c3aca06f174e8827ca6445a4353a1995584311487fc4e3e", size = 3518768, upload-time = "2025-09-22T04:04:57.097Z" }, ] +[[package]] +name = "mako" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2a/12/b5fa2353e2754cd67fb9f83793fa48ff42c213a5da7e719869d2301f6ab8/mako-1.4.1.tar.gz", hash = "sha256:d7904710b662996425a21627710c4777c45053146942cf8a7aebf757c92b8c27", size = 410165, upload-time = "2026-08-05T06:10:56.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/54/12ed58d458474aaab5c3d180173e745a4fe131bb330370596876d19ff60f/mako-1.4.1-py3-none-any.whl", hash = "sha256:a359d9a94a541213958742b2698d0a7757bb83551767bc468a74b9905aba9617", size = 80010, upload-time = "2026-08-05T06:10:58.248Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -1600,6 +1708,53 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/05/33/2d74d588408caedd065c2497bdb5ef83ce6082db01289a1e1147f6639802/psutil-5.9.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8", size = 249898, upload-time = "2024-01-19T20:47:59.238Z" }, ] +[[package]] +name = "psycopg" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/82/df3312c0ca083d5b43b352f27d4dd8b1e614bd334473074715d9e0000da4/psycopg_binary-3.3.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:612a627d733f695b1de1f9b4bd511c15f999a5d8b915d444bbd7dd71cf3370da", size = 4609813, upload-time = "2026-05-01T23:26:30.612Z" }, + { url = "https://files.pythonhosted.org/packages/1f/b5/d74d542458d3e8ac0571d8a88f57ca369999b9a82f4fa528052d0d7d3e4c/psycopg_binary-3.3.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:13a7f380824c35896dcac7fe0f61440f7ca49d6dc73f3c13a9a4471e6a3b302e", size = 4676799, upload-time = "2026-05-01T23:26:38.475Z" }, + { url = "https://files.pythonhosted.org/packages/09/67/06bab9c60671999f4c6ceff1b334f3ac1f9fc5789eb467c714623ea21de9/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:276904e3452d6a23d474ef9a21eee19f20eed3d53ddd2576af033827e0ba0992", size = 5497050, upload-time = "2026-05-01T23:26:47.061Z" }, + { url = "https://files.pythonhosted.org/packages/72/9b/023433e2b20f970de1e22d29132a95281277646da0b2e2879dd4ee94b8c1/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab8cca8ef8fb1ccf5b048ae5bd78ba55b9e4b5d472e3ce5ca39ff4d2a9c249e4", size = 5172428, upload-time = "2026-05-01T23:26:56.708Z" }, + { url = "https://files.pythonhosted.org/packages/08/cd/ae16da8fde228a38b2fe9269bbc13cf89e0186173f2265600f02d6a71e64/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7465bfe6087d2d5b42d4c53b9b11ca9f218e477317a4a162a10e3c19e984ba8e", size = 6762746, upload-time = "2026-05-01T23:27:07.023Z" }, + { url = "https://files.pythonhosted.org/packages/4f/81/0ba09fa5f5f88779093a2541a8e02489825721f258ab88058b11d68b3eb5/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:22cdbf5f91ef7bb91fe0c5757e1962d3127a8010256eefd9c61fcaf441802097", size = 5006033, upload-time = "2026-05-01T23:27:12.221Z" }, + { url = "https://files.pythonhosted.org/packages/73/6a/629136040cc3497adb442a305710b5913f2a754d4630fc3d3717c4c0df65/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e2631da29253a98bd496e6c4813b24e09a4fe3fb2a9e88513305d6f8747cce95", size = 4534175, upload-time = "2026-05-01T23:27:18.248Z" }, + { url = "https://files.pythonhosted.org/packages/7c/32/1027f843c6dc2d5d51960ee62cc0c2cf755a4c39455aff1371173edbef7d/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7f7668f30b9dd5163197e5cbf4e0efd54e00f0a859cc566ce56cfc31f4054839", size = 4224203, upload-time = "2026-05-01T23:27:24.3Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e1/380a724d9093c74adb14d4fce920ea8327838abb61f760b1448586b14a8e/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:cffc3408d77a27973f33e5d909b624cce683db5fc25964b02fe0aae7886c1007", size = 3954509, upload-time = "2026-05-01T23:27:30.815Z" }, + { url = "https://files.pythonhosted.org/packages/db/cd/895893ae575a09c97ccfd5def070d88993d955ef34df45a881fd5ff506d6/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0579252a1202cd73e4da137a1426e2dae993ae44e757605344282af3a082848c", size = 4259551, upload-time = "2026-05-01T23:27:38.828Z" }, + { url = "https://files.pythonhosted.org/packages/dd/c6/2330a20794e37a3ec609ef2fd8522919ec7a4395a1abf979a8e2d1775cd5/psycopg_binary-3.3.4-cp311-cp311-win_amd64.whl", hash = "sha256:41f2ec0fea529832982bcb6c9415de3c86264ebe562b77a467c0fbcd7efbba8d", size = 3572054, upload-time = "2026-05-01T23:27:45.455Z" }, + { url = "https://files.pythonhosted.org/packages/95/7d/03818e13ba7f36de93573c93ee3482006d3dfa8b0f8d28df511bad0a1a92/psycopg_binary-3.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089", size = 4591122, upload-time = "2026-05-01T23:27:56.162Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b9/11b341edf8d54e2694726b273fe9652b254d989f4f63e3ac6816ad6b55f4/psycopg_binary-3.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578", size = 4669943, upload-time = "2026-05-01T23:28:04.522Z" }, + { url = "https://files.pythonhosted.org/packages/8b/18/4665bacd65e7865b4372fcd8abb8b9186ada4b0025f8c2ca691b364a556c/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9", size = 5469697, upload-time = "2026-05-01T23:28:11.337Z" }, + { url = "https://files.pythonhosted.org/packages/7c/b1/b83136c6e510593d9b0c759ba5384337bc4ad82d19fda675adc4b2703c84/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d", size = 5152995, upload-time = "2026-05-01T23:28:20.529Z" }, + { url = "https://files.pythonhosted.org/packages/67/8d/a9821e2a648afe6091989929982a3b0f00b2631a859cb81379728f08fb75/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b", size = 6738180, upload-time = "2026-05-01T23:28:30.654Z" }, + { url = "https://files.pythonhosted.org/packages/7e/58/2e349e8d23905dc2317b80ac65f48fb6f821a4777a4e994a60da91c4850f/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070", size = 4978828, upload-time = "2026-05-01T23:28:37.277Z" }, + { url = "https://files.pythonhosted.org/packages/45/48/57b00d03b4721878326122a1f1e6b0a90b85bcaec56b5b2f8ea6cfa45235/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68", size = 4509757, upload-time = "2026-05-01T23:28:43.078Z" }, + { url = "https://files.pythonhosted.org/packages/25/37/33b47d8c007df69aec500df5889767c4d313748e8e9e27a2fef8a6dabcee/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16", size = 4190546, upload-time = "2026-05-01T23:28:50.016Z" }, + { url = "https://files.pythonhosted.org/packages/ca/c6/32b0835dbc2122617902b649d76a91c1e75406e76bf3d595b0c3bb5ffad6/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652", size = 3926197, upload-time = "2026-05-01T23:28:55.55Z" }, + { url = "https://files.pythonhosted.org/packages/cd/68/d190ef0c0c5b16ded07831dabc8ddd412f4cdab07ec6e30ed38d9bda0e1f/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e", size = 4236627, upload-time = "2026-05-01T23:29:05.336Z" }, + { url = "https://files.pythonhosted.org/packages/25/8f/81dcbc2e8454b74d14881275ea45f00791052dac531a9fa8be1730d1685b/psycopg_binary-3.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4", size = 3560782, upload-time = "2026-05-01T23:29:11.967Z" }, +] + [[package]] name = "ptyprocess" version = "0.7.0" @@ -2288,6 +2443,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/a0/bb38d3b76b8cae341dad93a2dd83ab7462e6dbcdd84d43f54ee60a8dc167/soupsieve-2.8-py3-none-any.whl", hash = "sha256:0cc76456a30e20f5d7f2e14a98a4ae2ee4e5abdc7c5ea0aafe795f344bc7984c", size = 36679, upload-time = "2025-08-27T15:39:50.179Z" }, ] +[[package]] +name = "sqlalchemy" +version = "2.0.52" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/21/77b4c147963073040dc3c3a5cb7a8c3001a1893c0209432cb77f9df836aa/sqlalchemy-2.0.52.tar.gz", hash = "sha256:5e2d46356ac2ccb7d268ab6c2319ac6a2b42f1b8d5fd8bd3d46855cd82abee97", size = 9945637, upload-time = "2026-08-11T19:07:09.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/08/cc5f7627b92f1456bc0b5fb7e98af4600248abe422a44da0d17a3fe6a448/sqlalchemy-2.0.52-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0c3ce43907374889f3352bdcc6195c970148a2cb71574cd0237a5071a37fb6c", size = 2172460, upload-time = "2026-08-11T20:58:22.429Z" }, + { url = "https://files.pythonhosted.org/packages/ed/dc/9a2abad8bfc8fdcd38c64adc056aeefab7aaa96ecd32f5e8c140e6375f17/sqlalchemy-2.0.52-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a0d48c4b80717c61385b4e966e087c839a66cfd7b780641dcb428f4dba65608", size = 3355720, upload-time = "2026-08-11T21:00:06.746Z" }, + { url = "https://files.pythonhosted.org/packages/a8/73/e75597b5841043e3c74055d00d4feb53d9a49a5c89ba2450d2d9aab53597/sqlalchemy-2.0.52-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:938325a5373267afc53bfbe72983b20fbd64ca47842aac62433c3da1137ecff1", size = 3354394, upload-time = "2026-08-11T21:05:51.454Z" }, + { url = "https://files.pythonhosted.org/packages/12/25/410fbc6c2f1fa8310f4ef1b6847d47d0ac1c042c7b4e81eaaca063d030a9/sqlalchemy-2.0.52-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5f8438a98d49424acf69d0d53c0a522951dfe49a6f2d86417fbb37ad3066ab43", size = 3306991, upload-time = "2026-08-11T21:00:08.603Z" }, + { url = "https://files.pythonhosted.org/packages/b2/ba/25ffd5c24681ea4b46e62c80ceca8200ce204de1773366321306cf3f608a/sqlalchemy-2.0.52-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4699dbb8d396d199e7e78fd4d525e3ad3d6008a9c8c0160b87e74c606c2c3736", size = 3327454, upload-time = "2026-08-11T21:05:53.368Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f1/0f1b1d4800e51218e736a06ed55a3b2a59c257600bbaca7673bf13d2dbec/sqlalchemy-2.0.52-cp311-cp311-win32.whl", hash = "sha256:cef328349452ae152637df4d11ce5a0919ecdf0a363e16c830c3518ee33bde72", size = 2131248, upload-time = "2026-08-11T21:09:50.765Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f0/04d2ac5ad66f3d31278f37064ed5f5ef3fe653f7bdaa67036663f223d186/sqlalchemy-2.0.52-cp311-cp311-win_amd64.whl", hash = "sha256:f1c850792a3b25a3ad74dade3f05e4f402cdebfea27438bcadafaa1617f77bcc", size = 2156943, upload-time = "2026-08-11T21:09:51.979Z" }, + { url = "https://files.pythonhosted.org/packages/e0/d5/1b77a026d161f98a08f11af1a5f6c47b98ee7c7e2648af525a1004826c78/sqlalchemy-2.0.52-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:be8c49131665dfe2cc74c498aa1240ffb548d0fd901325dd11c2c7a18956f727", size = 2170940, upload-time = "2026-08-11T20:58:11.25Z" }, + { url = "https://files.pythonhosted.org/packages/54/bd/f444444adb37b5d53753fb1730ee7a421628e2e3b756c4da461af7e6394a/sqlalchemy-2.0.52-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b2d9e507a458832adcfbd8af6e2036ddf069b7710b799448542ebccae2dceee", size = 3383415, upload-time = "2026-08-11T21:02:38.534Z" }, + { url = "https://files.pythonhosted.org/packages/be/57/2eadf93a552568c57e8680b7e58bb5e9770d80942a1bdbaf4f2f63f0d7c8/sqlalchemy-2.0.52-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8738008376d22f30f411ea3efecf39b51110b6996d80bb73786f30bcfdd5fd3b", size = 3398577, upload-time = "2026-08-11T21:16:59.092Z" }, + { url = "https://files.pythonhosted.org/packages/15/c3/2887cf9dd111d1fbf05d22165b404c221ef43e029f7a2695e7302f27a7cc/sqlalchemy-2.0.52-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37a4d548327b6cab9c7d8cdb4e0e82feabee0110c4d150059068e2d1cfbd99ee", size = 3328225, upload-time = "2026-08-11T21:02:40.183Z" }, + { url = "https://files.pythonhosted.org/packages/02/0f/466bdf9e1feeeef5587f868c187d8687e21ff8c85b1775e9041130181132/sqlalchemy-2.0.52-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e49f51a5d59857a7a0dcaf9469febf7197d9394bd88f00d69c2c4e848112cdbf", size = 3357374, upload-time = "2026-08-11T21:17:01.076Z" }, + { url = "https://files.pythonhosted.org/packages/22/20/5c2b4583904af4173076dda1c9e53c9e2ffc7a702d2efde0216bbacbf7cb/sqlalchemy-2.0.52-cp312-cp312-win32.whl", hash = "sha256:afda3ec521d0517d0de783fc70030775841900896d832de5bbd066549290470e", size = 2129366, upload-time = "2026-08-11T21:14:50.991Z" }, + { url = "https://files.pythonhosted.org/packages/ed/06/543dab8ef62d4e9fb96fb31a30c2b8b14a8763bccf48d428294d6b3041c0/sqlalchemy-2.0.52-cp312-cp312-win_amd64.whl", hash = "sha256:2d5e53e36e37129fe0be8b9d08b6e4052c10a963ee6cda56c8c10dcc194b99ca", size = 2157344, upload-time = "2026-08-11T21:14:52.453Z" }, + { url = "https://files.pythonhosted.org/packages/b3/3f/3582293d1e185e71d19d7c731c3e2ee20ba21981c4a1115c0806c1f62120/sqlalchemy-2.0.52-py3-none-any.whl", hash = "sha256:3b81b8363a919ce53453591cdb93702e6bd54ade6c4fa2f468fc053baee5ed89", size = 1950700, upload-time = "2026-08-11T20:47:21.603Z" }, +] + [[package]] name = "sse-starlette" version = "3.0.4" diff --git a/backend/worker.py b/backend/worker.py new file mode 100644 index 0000000..9077afc --- /dev/null +++ b/backend/worker.py @@ -0,0 +1,110 @@ +"""Durable worker loop. + +A standalone process that polls the durable ``jobs`` table, claims one job per +iteration, dispatches to a registered handler, and records completion or +failure. This is the portable core of the production worker topology and does +not depend on a specific broker; a queue provider (Redis/RabbitMQ) can be added +later by swapping the claim strategy without changing handlers. + +Run with, e.g.:: + PYTHONPATH=backend uv run python -m worker --once + PYTHONPATH=backend uv run python -m worker --poll-interval 1.0 +""" + +import argparse +import time +from typing import Callable, Optional + +from app.db import create_database_engine, create_session_factory +from app.services.job_queue import JobQueue + + +def _make_queue(): + engine = create_database_engine() + session_factory = create_session_factory(engine) + session = session_factory() + return engine, session, session_factory + + +def run_once(session, *, worker_id: str = "worker", organization_id: Optional[str] = None, **handlers) -> bool: + """Claim and process one job; returns True when one was handled.""" + queue = JobQueue(session) + for operation, handler in handlers.items(): + queue.register_handler(operation, handler) + job = queue.claim_next_job(worker_id=worker_id, organization_id=organization_id) + if job is None: + session.rollback() + return False + try: + result = queue.dispatch(job, payload=job.result) + queue.complete_job(job.id, result=result) + session.commit() + except Exception as exc: # noqa: BLE001 - worker must not die on a bad job + session.rollback() + try: + queue.fail_job(job.id, error_code=type(exc).__name__) + session.commit() + except Exception: # noqa: BLE001 + session.rollback() + return True + + +def run_loop( + *, + poll_interval: float = 1.0, + worker_id: str = "worker", + organization_id: Optional[str] = None, + handlers: Optional[dict[str, Callable]] = None, +) -> None: + handlers = handlers or {} + engine, session, _ = _make_queue() + try: + while True: + handled = run_once( + session, + worker_id=worker_id, + organization_id=organization_id, + **handlers, + ) + if not handled: + time.sleep(poll_interval) + else: + session = _fresh_session(engine, session) + finally: + session.close() + engine.dispose() + + +def _fresh_session(engine, session): + try: + session.close() + except Exception: # noqa: BLE001 + pass + return create_session_factory(engine)() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Durable job worker") + parser.add_argument("--once", action="store_true", help="process a single job and exit") + parser.add_argument("--poll-interval", type=float, default=1.0) + parser.add_argument("--worker-id", default="worker") + parser.add_argument("--organization-id", default=None) + args = parser.parse_args() + + if args.once: + engine, session, _ = _make_queue() + try: + run_once(session, worker_id=args.worker_id, organization_id=args.organization_id) + finally: + session.close() + engine.dispose() + else: + run_loop( + poll_interval=args.poll_interval, + worker_id=args.worker_id, + organization_id=args.organization_id, + ) + + +if __name__ == "__main__": + main() diff --git a/backend/wsgi.py b/backend/wsgi.py new file mode 100644 index 0000000..dc4f3f5 --- /dev/null +++ b/backend/wsgi.py @@ -0,0 +1,13 @@ +"""WSGI entry point for production (gunicorn). + +Usage: gunicorn -w 4 -b 0.0.0.0:5001 wsgi:app +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from app import create_app # noqa: E402 + +app = create_app() diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md new file mode 100644 index 0000000..56b1481 --- /dev/null +++ b/docs/HANDOFF.md @@ -0,0 +1,91 @@ +# HANDOFF — MiroFish SaaS Migration + +**Repo:** `/Users/kunthawat/Gitea/MiroFish` +**Branch:** `feat/saas-foundation` +**Base:** `89d04e7` +**Status:** SaaS foundation in progress; not production-ready; no commit/push yet + +## Current outcome + +The repository now has a tested identity/authentication foundation, tenant/owner guards around the main graph, simulation, report, and task lookup paths, durable operations schema/jobs/idempotency records, and a bounded local graph → profile → simulation → report persistence path. The original Zep runtime remains the default compatibility backend and legacy file-backed product persistence remains, so this is not yet a completed SaaS migration or production release. + +## Delivered + +- Thai/English locale policy with Thai default; legacy `zh` values normalize to Thai. +- Frontend API error sanitization and backend structured error envelopes. +- No full JSON request-body logging. +- SQLAlchemy models and Alembic migrations for organizations, users, memberships, and revocable sessions. +- Argon2 password hashing. +- HttpOnly session cookie plus signed CSRF cookie/header validation. +- Explicit credentialed CORS allowlist. +- Three roles only: `super_admin`, `admin`, `user`. +- `/api/auth/login`, `/api/auth/me`, `/api/auth/logout`. +- `/api/admin/users` list/create with role escalation prevention and response redaction. +- Project `organization_id`/`owner_user_id` metadata and scoped project reads/lists. +- Blueprint-wide auth and CSRF checks for simulation/report routes. +- Fail-closed scope checks for project, graph, simulation, report, and task IDs in path/query/body. +- Cross-tenant regression tests for project/simulation/report/task access. +- Frontend auth store using cookie sessions, `/login` view, protected workspace router guard, and localized Thai/English login copy. +- Frontend report-status client corrected to POST `{ task_id, simulation_id }`. +- Durable local memory tables and a tenant-scoped SQLAlchemy repository. +- Durable `jobs`, idempotency records, and audit-log schema with migrations `0004_operations` through `0006_job_metadata`; `TaskManager` binds to the current app/session factory, fails closed when an app context has no factory, and has SQL-level tenant/owner filters. +- SQLite engines created by the shared DB helper enable foreign-key enforcement; regression coverage verifies invalid organization/owner references plus `SET NULL`/`CASCADE` delete behavior. +- GraphBuilder async adapter now carries organization/session context into durable task metadata. +- Deterministic local graph → profile → simulation-config → report-tools → persisted-report regression. +- Strict Pydantic extraction schema, bounded prompts, JSON validation, and deterministic LLM-result ingestion service. +- Golden parity fixtures and local adapters for entity filtering/context, quick search, panorama temporal inventory, and insight entity/relationship output shapes. +- Explicit `MEMORY_BACKEND=local` path now serves simulation entity routes and preparation; `SimulationManager.create_entity_reader()` fails closed when local mode has no injected scoped factory; default remains Zep during migration. Fresh bounded reviewer `deleg_029828b6` independently passed this exact reader/fallback slice with no security concerns or logic errors. +- The bounded local panorama/insight/entity-reader parity slice is covered by RED/GREEN tests; conflicting graph IDs fail closed and local simulation activity updater persists tenant-scoped episodes without constructing a Zep client. +- API safe-error regression gate covers simulation, graph, report, agent-group, and template boundaries; task/project/report/run state no longer persists raw exception text. + +## Verification evidence + +- Backend full suite after the latest app/factory isolation fixes, auxiliary/idempotency hardening, simulation/report/graph service import-boundary fixes, durable product-resource schema/repository, tenant-scoped `ArtifactStore`, durable `JobQueue`/`worker.py`, versioned redacted `PlatformSettings`, durable `RateLimiter`, durable LLM `UsageService`, durable redacted `AuditService`, and durable single-use `PasswordResetService`: **193 passed** with `PYTHONPATH= uv run pytest -q` from `backend` (2026-08-24); twenty-four Alembic deprecation warnings remain. +- Schema regression: **16 passed**, including Alembic upgrade → downgrade `0003_memory` → re-upgrade, operation-table column/index/FK inspection, SQLite PRAGMA enforcement, invalid org/owner rejection, delete actions, cross-app TaskManager binding checks, and background use without Flask context. +- Auxiliary API security regression: **8 passed**; every template/agent-group route requires auth, LLM mutations require CSRF + idempotency, safe internal errors are redacted, and both LLM mutations replay completed responses. +- Idempotency API regression: **5 passed**; same-route replay/conflict semantics remain, same-key cross-route/body replay is rejected, and multipart file content/repeated-file coverage is verified. +- Local consumer-boundary regression: **9 fresh-import tests passed** (`test_local_service_import.py`); fresh local imports of the updater, `SimulationManager`, `SimulationRunner`, `ReportAgent`, and `GraphBuilderService` do not eagerly load `zep_cloud`, `add_activity_from_dict` uses the shared activity contract without importing `zep_cloud` on the local path, and the reviewer-requested regression guards for explicit Zep-path imports, shared `AgentActivity` identity, and the `graph_builder.Zep` seam were added and pass. The activity-dict boundary regression also passes. +- Durable product-resource persistence: **21 product-persistence tests passed** (`test_product_persistence.py`). Migration `0007_product_resources` adds tenant/owner-scoped `projects`, `simulations`, and `reports` tables with FK cascade/SET-NULL, indexes, and a migration round-trip + `command.check` (no autogenerate drift). `ProductRepository` provides flush-only tenant-scoped create/get/list for all three with cross-tenant isolation verified. `sync_project`/`sync_simulation`/`sync_report` are idempotent dual-write bridges wired into project-creation, simulation-creation, and report-generation paths; `SimulationState` carries tenant scope and `_save_simulation_state` mirrors every status update into the durable table when a session factory is available, fail-closed when unscoped. `scoped_project` reads the durable table first (mapping back to the legacy `Project` shape) when the local backend is active, then falls back to filesystem, keeping existing routes intact. Read cutover/dual-write are gated to the local backend so the legacy Zep filesystem path is undisturbed. +- Tenant-scoped artifact store: **12 artifact-store tests passed** (`test_artifact_store.py`). `ArtifactStore` resolves tenant-scoped paths under a configured root, rejects traversal/absolute components (fail-closed), and exposes a store/read/exists/delete interface (`default_artifact_store()` is filesystem-backed; swapable for object storage later). +- Durable job queue + worker core: **10 tests passed** (`test_job_queue.py` 7, `test_worker.py` 3). `JobQueue` claims/executes/completes/fails durable `jobs` with optional organization scoping, a handler-registry dispatch (raises on unhandled operations for the worker to fail), and no double-claim; `worker.py` `run_once`/`run_loop` provide the portable worker entry point (no broker dependency yet). +- Versioned, redacted LLM settings: **4 tests passed** (`test_settings_service.py`). Migration `0008_platform_settings` + `SettingsService` store versioned global platform settings; the API key is encrypted with a `SECRET_KEY`-derived Fernet key (never stored/logged in plaintext, never in job snapshots), public reads return a masked value, and `snapshot_for_job()` captures the settings version for reproducible work. `cryptography` added as a direct dependency. +- Durable rate limiting: **6 tests passed** (`test_rate_limiter.py`). Migration `0009_rate_limit` + `RateLimiter` implement a durable sliding-window counter (survives restarts, per-key, no secrets), and are wired into the `/api/auth/login` route so excessive attempts return 429 (`api.tooManyAttempts` added in `th`/`en`). +- Durable LLM usage accounting: **4 tests passed** (`test_usage_service.py`). Migration `0010_usage_events` + `UsageService` record per-org/user LLM usage (model, input/output tokens, estimated cost) without storing prompt content or secrets, with org-scoped listing and total-cost totals. +- Durable redacted audit service: **3 tests passed** (`test_audit_service.py`). `AuditService` records tenant-scoped audit events while stripping sensitive keys (secrets, tokens, api keys, prompts) from details, and is wired into the `/api/auth/login` success path (`auth.login` event). +- Durable single-use password reset: **6 tests passed** (`test_password_reset.py`). Migration `0011_password_reset_tokens` + `PasswordResetService` issue expiring, single-use, hashed reset tokens (plaintext never stored), bump `auth_version` on consume to invalidate old sessions, and are exposed as `/api/auth/password-reset/request` + `/api/auth/password-reset/confirm` (enrollment-agnostic request prevents account enumeration; also covers invite-pending password setup). +- Frontend contract tests: **10 passed**; `npm run build`: **PASS**; existing chunk-size/dynamic-import warnings remain. +- Python `compileall`: **PASS**. +- Diff security scan: no introduced `shell=True`, `eval/exec`, or pickle loading in backend code; `git diff --check`: **PASS**. +- `ruff` is not installed in the backend environment; lint is skipped, not treated as a pass. +- Dev server `/login` smoke response: **PASS**. Real 320×568 and 500×768 browser capture is **not verified** because Chrome requested an OS remote-debugging permission and no permission dialog was clicked. + +## Important incomplete work + +1. **Zep replacement is not complete.** Local memory storage/extraction contracts and a bounded graph → profile → simulation → report persistence regression exist, but semantic LLM parity and full consumer cutover are not complete; Zep remains the default compatibility backend. +2. **Product persistence foundation is in place.** Durable tenant/owner-scoped `projects`/`simulations`/`reports` schema (migration `0007_product_resources`), a scoped `ProductRepository`, dual-write for creation and simulation run status, `scoped_project` read-first cutover (local backend), and a tenant-scoped `ArtifactStore` abstraction are added and tested. Legacy `ProjectManager`/`SimulationManager`/`ReportManager` filesystem paths remain authoritative-by-default for the Zep backend and for artifact payloads; full read cutover for simulation/report and migrating artifact payloads to object storage are deferred to the production storage/worker phase. +3. **Production worker/queue topology core is in place; broker/storage not decided.** A durable `JobQueue` (claim/dispatch/complete/fail, tenant-scoped) and a portable `worker.py` run-loop entry point exist and are tested, alongside a tenant-scoped `ArtifactStore`. Production decisions remain: choose a broker (Redis/RabbitMQ) or keep polling the durable table, choose object storage, and define the WSGI/worker/supervisord deployment topology. Background-thread execution in routes is not yet moved onto the durable queue. +4. **Admin user-management + super-admin settings UI base in place.** Backend `GET/POST/PATCH /api/admin/users` (role/status policy-guarded, status super-admin only) and `GET/PUT /api/admin/settings` (super-admin only, secret encrypted+masked via `SettingsService`). Frontend `AdminView.vue` (user mgmt) and `SettingsView.vue` (LLM settings form) are routed at `/admin` and `/admin/settings` with admin/super-admin role guards and th/en i18n. Invite self-setup UX, a connection-test endpoint, and full i18n/mobile review remain. +5. **LLM settings storage base is in place; admin/bootstrap UI not built.** A versioned, encrypted `PlatformSettings` store (`SettingsService` + migration `0008`) exists and is tested. Super-admin settings UI, provider preset validation, and connection-test endpoints are not yet built; active runtime still reads environment variables. +6. **Frontend active UI i18n gates pass; legacy `Process.vue` retired.** Frontend contract tests **10 passed** and the production build contains no CJK code points. The unused legacy `Process.vue` (Chinese copy) was retired — the router's `name: 'Process'` uses `MainView.vue`. Real mobile 320×568/500×768 visual capture requires a full-stack + authenticated browser session and is not yet verified; a standalone `vite preview` of the SPA renders a blank mount (needs backend bootstrap), which is not treated as a pass. +7. **Docker deployment drafted as production multi-service; container build not yet proven.** Dockerfile is rebuilt as a production multi-service image (multi-stage frontend build, python + gunicorn `wsgi:app`, nginx serving the built SPA with history fallback and `/api`→gunicorn proxy, supervisord running backend + durable worker). `backend/wsgi.py` and `gunicorn>=21` were added. Local smoke test (gunicorn + built-dist static serving) passed — `/health` OK, assets rebuilt — but the nginx SPA-fallback + proxy + worker-poll path must be verified by building the image in EasyPanel; no Docker runtime is available on this Mac. +8. **SaaS hardening mostly complete.** Durable rate limiting (wired to login), durable LLM usage/cost accounting, a durable redacted audit service, and durable single-use password reset (which also covers invite-pending setup) are added and tested. Still open: a complete resource authorization matrix across every role × endpoint, a full retryable-mutation inventory, and long-running-job review. +9. **Auxiliary API hardening is verified.** Fresh reviewer `deleg_d128494e` returned `passed=true` with empty `security_concerns` and `logic_errors`; current auxiliary security tests pass **8/8**. This closes only the template/agent-group slice, not the broader SaaS gate. +10. **Bounded schema/TaskManager review passed.** Fresh reviewer `deleg_fb0f80b0` returned `passed=true` with empty `security_concerns` and `logic_errors` after current-code runtime app/factory mismatch probes, SQLite pooled/NullPool foreign-key checks, ORM/migration parity, and migration round-trip checks. Its suggestions are non-blocking: expand mismatch coverage across every TaskManager operation, broaden pooled/NullPool assertions in tests, and assert full ORM/migration signatures. This closes only the exact schema/TaskManager slice; broad reviewer `deleg_949ab2f4` timed out and the wider SaaS gate remains open. +11. **The current bounded local import-boundary review passed.** Fresh reviewer `deleg_65bce3de` returned `passed=true` with empty `security_concerns` and `logic_errors`; it ran the focused boundary command with **6 passed** and isolated probes for five fresh local imports, shared `AgentActivity`, explicit Zep paths, the `graph_builder.Zep` seam, and subprocess isolation. This closes only the bounded import-boundary slice; semantic parity, complete consumer cutover, and the wider SaaS gate remain open. + +## Next implementation order + +1. Keep the current staged snapshot and re-run verification after any further edits. +2. Extend golden parity and tenant-negative coverage to the remaining local consumers and simulation-context/report E2E path. +3. Switch graph builder, entity reader, tools, profile generator, and report agent to the local memory adapter. +4. Remove Zep dependency/config only after parity and migration tests pass. +5. Add admin/super-admin UI and settings policy. +6. Finish production worker/deployment hardening and run a fresh end-to-end tenant matrix. + +## Guardrails + +- Exactly three role identifiers: `super_admin`, `admin`, `user`. +- Browser never receives LLM/API secrets. +- Backend authorization is authoritative; frontend guards are convenience only. +- Do not claim semantic LLM/Zep parity without golden-fixture evidence. +- Do not commit or deploy while the independent review gate remains unresolved. diff --git a/docs/engineering-log.md b/docs/engineering-log.md new file mode 100644 index 0000000..1fbce63 --- /dev/null +++ b/docs/engineering-log.md @@ -0,0 +1,32 @@ +# Engineering Log + +## Current status + +| Milestone | Status | Last verified | Evidence | Next action | +|---|---|---|---|---| +| Baseline architecture study | complete | 2026-08-23 | `npm run build` passed; `compileall` passed; `git diff --check` passed; source inventory completed | Review MiroFish SaaS plan and lock M0 decisions | +| Thai/English frontend hardening | bounded active-surface gate | 2026-08-24 | Frontend contract tests **10 passed**; production build passed; active routed UI has no hardcoded CJK visible strings; legacy unused `Process.vue` (Chinese copy) retired — router uses `MainView.vue`; mobile viewport capture not yet verified | Migrate remaining legacy views and perform real 320×568 and 500×768 capture when browser permission is available | +| Zep replacement | bounded local E2E slice | 2026-08-24 | Local graph → profile → simulation config → report tools → persisted report regression passed; default remains Zep; no full consumer cutover or semantic parity claim | Cut over remaining consumers and close semantic/E2E gaps | +| Auth/tenant/roles | bounded foundation | 2026-08-24 | Identity/session/roles/CSRF/CORS/idempotency/resource guards covered by focused tests; durable task app-state leak fixed; task query filters now push tenant predicates into SQL | Complete broader tenant matrix, admin UI, rate limits, audit/usage policy | +| SaaS foundation batch | in progress | 2026-08-24 | Backend full suite **193 passed** after app/factory isolation, SQLite-FK, auxiliary API auth/CSRF/idempotency, cross-route/multipart idempotency, local consumer-boundary fixes, durable product-resource schema/repository, tenant-scoped `ArtifactStore`, durable `JobQueue`+`worker.py`, versioned redacted `PlatformSettings`, durable `RateLimiter` (wired to login), durable LLM `UsageService`, durable redacted `AuditService`, and durable single-use `PasswordResetService` + endpoints (also covers invite-pending setup); schema/TaskManager regression **16 passed**; auxiliary security **8 passed**; idempotency API **5 passed**; local import-boundary regression **9 fresh-import tests**; product-resource persistence **21 tests**; artifact store **12 tests**; job queue/worker **10 tests**; settings service **4 tests**; rate limiter **6 tests**; usage service **4 tests**; audit service **3 tests**; password reset **6 tests**; frontend gates passed; bounded reviewers passed their exact slices; hardened bases ready; remaining: resource authz matrix completion, admin/bootstrap UI, and deploy topology; `ruff` unavailable; no commit/push/deploy | Complete admin UI, authz matrix, then deploy topology; do not claim full-system approval | +| Admin/super-admin UI | bounded foundation | 2026-08-24 | Backend: `GET/POST/PATCH /api/admin/users` + `GET/PUT /api/admin/settings` (super-admin only, masked/encrypted secret via `SettingsService`); Frontend: `AdminView.vue` (user mgmt) + `SettingsView.vue` (LLM settings form) routed at `/admin` + `/admin/settings` with admin/super-admin role guards, th/en i18n identical; build + 10 frontend tests pass; backend 197 passed | Add invite self-setup UX, connection-test endpoint, then full i18n/mobile review | +| Production worker/deployment | production topology drafted, locally smoke-tested | 2026-08-24 | Dockerfile rebuilt as multi-stage production (frontend build + python-gunicorn + nginx-SPA-proxy + supervisord worker); `backend/wsgi.py` gunicorn entry + `gunicorn>=21` added; local smoke test: gunicorn `wsgi:app` started, `/health` OK, `/api/auth/login` 401, built SPA assets served 200; backend 197 passed | Build in EasyPanel container to verify nginx SPA-fallback + `/api` proxy + worker poll; choose broker (Redis vs durable-poll) + object storage for full readiness | + +## Guardrails + +- No production implementation was changed during the architecture study. +- Keep exactly three role identifiers: `super_admin`, `admin`, `user`. +- Never expose LLM/API secrets to the browser or commit them to docs/logs. +- Backend authorization and tenant scope are authoritative; frontend visibility is not security. +- Do not claim LLM/Zep semantic parity without golden-fixture evidence. + +## Entry index + +- `docs/engineering-log/2026-08-23-architecture-study.md` +- `docs/test-evidence/2026-08-23-baseline.md` +- `.hermes/plans/2026-08-23_110451-mirofish-saas-migration.md` +- `docs/engineering-log/2026-08-24-simulation-memory-fallback.md` +- Independent SaaS audit findings were incorporated after the initial plan: raw log redaction, ID/path confinement, idempotency, upload drafts and API contract tests. +- Safe-error audit now covers API response/persisted task state paths; raw exception/traceback values are replaced with generic localized errors and error-type-only server logs. +- `docs/engineering-log/2026-08-24-memory-parity-panorama-insight.md` records the bounded parity implementation, verification evidence, review verdict, and remaining semantic/E2E boundary. +- `docs/engineering-log/2026-08-24-saas-foundation-verification.md` diff --git a/docs/engineering-log/2026-08-23-architecture-study.md b/docs/engineering-log/2026-08-23-architecture-study.md new file mode 100644 index 0000000..180e623 --- /dev/null +++ b/docs/engineering-log/2026-08-23-architecture-study.md @@ -0,0 +1,42 @@ +# Architecture Study — MiroFish SaaS Migration + +**Date:** 2026-08-23 (+07:00) +**Repository:** `/Users/kunthawat/Gitea/MiroFish` +**HEAD:** `89d04e7` +**Status:** complete — plan only; no production source changes + +## Scope + +Reviewed frontend i18n/rendering, backend API/entry points, Zep integration, LLM prompt paths, persistence, auth/security posture, role/tenant requirements and Docker runtime. + +## Evidence + +- Working tree was clean at start and after baseline verification. +- `npm run build` passed; Vite emitted chunk-size and `pendingUpload.js` import warnings. +- `python3 -m compileall -q backend/app backend/run.py backend/scripts` passed. +- `git diff --check` passed. +- Backend route inventory: 64 routes; no auth middleware found. +- Frontend locale inventory: `en` and `th` have matching 648 flattened keys; `zh` is still present and is the fallback/default path. +- Current built frontend still contains CJK code points in generated HTML/JS. +- Zep usage was traced through graph build, entity reader, search tools, graph memory updater, profile generation and ReportAgent. +- Current project/task/simulation/report state is mostly filesystem/in-memory scoped globally, not by user or organization. +- Independent SaaS audit confirmed additional risks: raw request-body logging, raw exception/path exposure, non-idempotent frontend retries, global history/debug routes, process-local pending uploads and a report-status HTTP method/body mismatch. + +## Decisions captured in plan + +- Use PostgreSQL as SaaS system of record; add tenant scope from v1. +- Use LLM for structured memory extraction/merge/summary/query reasoning, not for durable storage or unbounded search. +- Preserve internal memory result contracts while replacing Zep implementation. +- Use exactly `super_admin`, `admin`, `user` with backend-enforced policy. +- Default product locale recommendation: Thai, with English switcher. +- Move long-running work to durable workers before calling the system production SaaS-ready. + +## Risks/open decisions + +- Need user decision on whether user-uploaded Chinese content must also be translated/hidden. +- Need user decision on multi-organization scope, invite flow, Zep data migration and LLM settings scope. +- Python runtime on the current host is 3.14 while project metadata requires Python 3.11–3.12; compile success is not full runtime proof. + +## Exact next action + +Get M0 answers, then create PR-sized implementation batches starting with persistence/auth contracts and locale/API error contracts. Do not begin by deleting Zep files without first implementing the local memory interface and storage schema. diff --git a/docs/engineering-log/2026-08-23-local-memory-first-consumer.md b/docs/engineering-log/2026-08-23-local-memory-first-consumer.md new file mode 100644 index 0000000..8d97f32 --- /dev/null +++ b/docs/engineering-log/2026-08-23-local-memory-first-consumer.md @@ -0,0 +1,28 @@ +# Local Memory First Consumer — 2026-08-23 + +## Scope + +Enabled the first explicit local-memory execution path without silently changing the existing Zep default. + +## Changes + +- Added `MEMORY_BACKEND=zep|local`; unknown values fail closed. +- Added per-request local entity-reader factory scoped by `organization_id` and `graph_id`. +- Added worker-owned SQLAlchemy session lifecycle for local readers. +- `SimulationManager` now accepts `entity_reader_factory` and uses it during preparation. +- `/api/simulation/entities/*` selects the configured reader and closes it after each request. +- `/api/simulation/prepare` uses the configured reader for preview and background preparation. +- `OasisProfileGenerator` supports `use_zep_context=False`; local mode does not initialize Zep context for profile enrichment. +- Default remains `MEMORY_BACKEND=zep` until remaining consumers and data migration are ready. + +## Evidence + +- Switch/route tests: 3 passed. +- Reader factory/entity parity/injection tests: passed. +- Full backend suite after this slice: **55 passed**. +- Frontend suite: **10 passed**. +- Build, compileall, and Alembic upgrade/check: passed. + +## Boundary + +This is not a full Zep removal. Profile generation still uses the legacy flow outside the explicit local preparation path, and graph tools/report consumers still need migration. diff --git a/docs/engineering-log/2026-08-23-local-memory-foundation.md b/docs/engineering-log/2026-08-23-local-memory-foundation.md new file mode 100644 index 0000000..34e0223 --- /dev/null +++ b/docs/engineering-log/2026-08-23-local-memory-foundation.md @@ -0,0 +1,32 @@ +# Local Memory Foundation — 2026-08-23 + +## Scope + +Started the Zep replacement without switching existing consumers yet. The goal of this slice is to separate LLM semantic extraction from durable storage, search, and authorization. + +## Delivered + +- SQLAlchemy memory tables: `memory_graphs`, `memory_episodes`, `memory_nodes`, `memory_edges`. +- Alembic revision `0003_memory` with SQLite upgrade/check parity. +- `SqlAlchemyMemoryRepository` constrained by `organization_id` + `graph_id`. +- Deterministic node normalization, confidence bounds, graph/node scope checks, and bounded search. +- Golden parity fixture and local adapters for entity filtering/context and quick search output shapes. +- Strict Pydantic extraction models with `extra="forbid"`. +- Bounded Thai/English extraction prompt builder. +- JSON/code-fence parser with validation. +- `MemoryExtractionService` that calls a JSON LLM client, validates output, resolves entity references deterministically, and persists only through the scoped repository. + +## Evidence + +- Memory repository tests: 2 passed. +- Extraction contract tests: 3 passed. +- Memory service test: 1 passed. +- Full backend suite: 47 passed. +- Frontend contract suite: 10 passed. +- Vite build: pass. +- Python compileall: pass. +- Alembic revisions `0001_identity` → `0003_memory` upgrade/check: pass. + +## Deliberate non-change + +Existing `GraphBuilderService`, `ZepEntityReader`, `ZepToolsService`, `OasisProfileGenerator`, and `ReportAgent` still use Zep. Consumer cutover is deferred until golden parity fixtures exist for entity reads, search, temporal edges, and simulation context. diff --git a/docs/engineering-log/2026-08-23-saas-foundation-continuation.md b/docs/engineering-log/2026-08-23-saas-foundation-continuation.md new file mode 100644 index 0000000..665bd53 --- /dev/null +++ b/docs/engineering-log/2026-08-23-saas-foundation-continuation.md @@ -0,0 +1,46 @@ +# SaaS Foundation Continuation — 2026-08-23 + +## Scope + +Continued implementation after reviewer delegation infrastructure failed to return a verdict. + +## Changes + +- Added blueprint-wide authentication and CSRF enforcement to simulation/report APIs. +- Added fail-closed resource helpers for project, graph, simulation, report, and task IDs. +- Scoped simulation/report listing and ownership checks to the authenticated actor. +- Added task metadata ownership for report-generation jobs and blocked cross-tenant task status reads. +- Added local `ApiError` handlers for simulation/report blueprints; removed the frozen exception behavior that caused `FrozenInstanceError` while unwinding a session context. +- Corrected frontend report-status method/body contract. +- Added cookie-session auth store, public login route, protected workspace router guard, and localized login view. + +## Evidence + +- Resource auth tests: 3 passed. +- Full backend suite: 41 passed. +- Frontend contract suite: 10 passed. +- Vite build: pass; existing chunk/import warnings remain. +- Python compileall: pass. +- Alembic SQLite upgrade/check: pass. +- Staged diff whitespace check: pass after staging the continuation changes. + +## Prevention notes + +- Any route that accepts a resource ID must pass through a shared scope helper before touching filesystem or in-memory state. +- Any task exposed through an API must carry `organization_id` and `owner_user_id`; legacy tasks without metadata fail closed. +- `ApiError` must remain a normal mutable exception class; frozen dataclasses cannot safely unwind through context managers because Python assigns traceback fields. +- Frontend API contract tests should assert HTTP method and request body against the backend route, not only function existence. + +## Remaining blocker + +Independent reviewer delegation exited before returning the required JSON verdict. This is recorded as `VERIFICATION BLOCKED`, not as an approval or security finding. + +## Follow-up — local cutover and safe-error gate + +- Added local graph activity updater coverage and graph → profile → report golden contract coverage. +- Replaced raw exception text in API response paths and persisted simulation/report/project/task/run state with generic localized errors; server logs retain only operation, identifier, exit code where needed, and exception type. +- Added `backend/tests/test_api_no_raw_exception_details.py` as a source-level regression gate. +- Verification: backend **67 passed**, frontend Node contract tests **10 passed**, Vite build **PASS**, Python compileall **PASS**, `git diff --check` **PASS**. +- `npm test` was attempted but the repository has no `test` script; the supported `node --test tests/*.mjs` command passed. +- Reviewer status remains `VERIFICATION BLOCKED` until the fresh independent reviewer returns a readable schema-valid terminal verdict. +- `deleg_289b16ce` also timed out without a terminal verdict after broad read-only inspection. Its intermediate targeted-test failure was not reproducible on the current tree: `tests/test_api_no_raw_exception_details.py` reran **1 passed**. Reviewer `deleg_e0e7bac1` was narrowed via steering and remains the active pending gate. diff --git a/docs/engineering-log/2026-08-24-graph-scope-review.md b/docs/engineering-log/2026-08-24-graph-scope-review.md new file mode 100644 index 0000000..06c90bf --- /dev/null +++ b/docs/engineering-log/2026-08-24-graph-scope-review.md @@ -0,0 +1,35 @@ +# Graph Scope Review — 2026-08-24 + +## Scope +Closed the two blockers from the earlier independent review: + +1. Project deletion must be organization-scoped and owner-scoped for regular users. +2. Graph read/delete routes must use the local adapter when `MEMORY_BACKEND=local`, without constructing or silently falling back to Zep. + +## Changes verified + +- `backend/app/api/graph.py` resolves the authenticated actor and routes project/graph access through scoped helpers. +- `backend/app/models/project.py` requires organization scope for project deletion and applies the user-owner restriction where applicable. +- `backend/app/services/local_graph_builder.py` requires organization and project scope and passes graph operations to the tenant-scoped repository. +- `backend/tests/test_graph_backend_routes.py` covers backend dispatch, cross-tenant project/graph behavior, and local graph deletion with nodes and an edge. + +## Verification evidence + +- Focused regression set: **15 passed**. +- Full backend suite: **72 passed**. +- Frontend contract tests: **10 passed**. +- Frontend production build: **PASS**; existing chunk-size/dynamic-import warnings remain. +- Python `compileall`: **PASS**. +- Added-lines security scan: **0 findings** for hardcoded secrets, shell injection, eval/exec, unsafe pickle, and interpolated SQL patterns. +- `git diff --check`: **PASS**. +- `ruff`: unavailable in the current backend environment; no lint verdict claimed. + +## Review gate + +Independent review attempts before this entry did not return readable terminal JSON and therefore were fail-closed. Fresh narrow read-only reviewer `deleg_dec1c4b0` then returned the required exact five-key JSON with `passed=true`, empty `security_concerns`, and empty `logic_errors` after reviewing all seven target files. The bounded blocker gate is closed. This does not claim approval, parity completion, production readiness, or full Zep removal. + +## Remaining work + +- The bounded graph/project/local-memory review gate is closed; continue the staged local-memory consumer cutover and durable resource work. +- Broader migration review, parity, production hardening, and end-to-end tenant matrix remain open. +- No commit, push, or deploy was made during this verification. diff --git a/docs/engineering-log/2026-08-24-memory-parity-panorama-insight.md b/docs/engineering-log/2026-08-24-memory-parity-panorama-insight.md new file mode 100644 index 0000000..d3bb5e5 --- /dev/null +++ b/docs/engineering-log/2026-08-24-memory-parity-panorama-insight.md @@ -0,0 +1,40 @@ +# Local Memory Parity Slice — 2026-08-24 + +## Scope + +Bounded local-memory compatibility work only: + +- `LocalMemoryTools.panorama_search` now preserves the legacy all-graph inventory contract while relevance-ranking active and historical facts. +- `LocalMemoryTools.insight_forge` now derives entity insights and relationship chains from matching edges with the legacy-compatible output shape. +- `LocalEntityReader` rejects explicitly supplied graph IDs that differ from its repository-bound graph ID. +- `MEMORY_BACKEND=zep` remains the default compatibility path; this slice does not remove Zep or cut over ReportAgent/ProfileGenerator globally. + +## TDD evidence + +1. Added `backend/tests/fixtures/memory_parity/tools_fixture.json` with three entities, current and invalidated temporal edges, and one unrelated active edge. +2. Added `backend/tests/test_memory_parity.py` covering panorama inventory/temporal behavior, insight entity/relationship shape, and five conflicting graph-ID call forms. +3. Ran the new tests before the production change; the suite failed for the expected missing parity and scope behaviors. +4. Implemented the minimal adapter and reader changes. +5. Re-ran the focused parity suite: **8 passed**. + +## Verification + +- Focused consumer regression set: **17 passed**. +- Backend full suite: **81 passed**. +- Python `compileall`: **PASS**. +- `git diff --check`: **PASS**. +- Added-lines security scan: **0 findings**. +- `ruff`: unavailable in the backend environment; no lint pass is claimed. +- A probe run from repository root failed because that cwd cannot resolve the backend pytest executable; the same probe rerun from `backend/` passed 8/8. This is a command-location issue, not a test failure. + +## Independent review gate + +- The first bounded reviewer was stopped after exceeding the context/time budget; its silence was not treated as approval. +- Replacement reviewer: `deleg_73fac0de`. +- Required five-key JSON verdict: `passed=true`, `security_concerns=[]`, `logic_errors=[]`. +- Non-blocking suggestion: consider future LLM query decomposition for `insight_forge`; local `sub_queries=[query]` remains an explicit semantic gap outside this slice. +- This closes the review gate for this slice only. It is not approval of full semantic parity, the broader migration, or production readiness. + +## Remaining boundary + +This proves only the bounded panorama/insight/entity-reader contract. It does not prove semantic LLM parity (`sub_queries` remains local `[query]`), complete graph/profile/simulation/report E2E, all local consumer cutovers, migration parity, or production readiness. diff --git a/docs/engineering-log/2026-08-24-saas-foundation-verification.md b/docs/engineering-log/2026-08-24-saas-foundation-verification.md new file mode 100644 index 0000000..269cc61 --- /dev/null +++ b/docs/engineering-log/2026-08-24-saas-foundation-verification.md @@ -0,0 +1,46 @@ +# SaaS Foundation Verification — 2026-08-24 + +**Branch:** `feat/saas-foundation` +**Status:** in progress; not production-ready; no commit/push/deploy + +## Milestone + +Closed the confirmed TaskManager app-state leak and extended the durable-task tenant boundary. Added a deterministic local end-to-end regression covering graph → entity reader → profiles → simulation config → report tools → persisted report. + +## Changes + +- `TaskManager` now binds to the current Flask app's session factory at construction time instead of relying on app-factory global singleton state. +- In-memory TaskManager fallback remains test/legacy-only and isolated from app-bound managers; an app context without its configured factory now fails closed instead of using the class-global binding. +- Shared SQLite engine creation enables `PRAGMA foreign_keys=ON` for every connection, including migration/test engines. +- Durable `get_task()` and `list_tasks()` accept organization/owner filters and push predicates into SQLAlchemy queries. +- Scoped task lookup and graph task listing pass tenant/owner predicates before the defensive metadata check. +- Legacy asynchronous `GraphBuilderService` accepts session/tenant context and writes it into durable task metadata; authenticated graph build passes the captured context into the worker adapter. +- Added `test_local_graph_profile_simulation_report_is_persisted` and `test_graph_builder_async_task_keeps_organization_scope`. + +## Verified evidence + +- TDD regression sequence: app-context fallback, SQLite-FK, reused-manager, explicit-mismatch, prebound-manager, background-use, auxiliary API auth/CSRF/idempotency, cross-route idempotency, multipart-content, and local Zep-import-boundary tests were observed RED where applicable, then passed GREEN; the full backend suite passed afterward (**123 passed**, four Alembic deprecation warnings). +- Schema regression: **16 passed**, including migration round-trip/model alignment, SQLite foreign-key enforcement, invalid org/owner rejection, delete actions, cross-app TaskManager binding, and background use without Flask context. +- Auxiliary API regression: **8 passed**; all five routes are authenticated, LLM mutations enforce CSRF/idempotency, safe error responses are redacted, and completed mutation responses replay correctly. +- Idempotency regression: **5 passed**; the request fingerprint separates method/path/query/body and includes multipart file content hashes across repeated file fields. +- Local consumer-boundary regression: **5 fresh-import tests passed** plus the activity-dict boundary regression; fresh subprocess tests prove `app.services.local_graph_memory_updater`, `SimulationManager`, `SimulationRunner`, `ReportAgent`, and `GraphBuilderService` do not eagerly load `zep_cloud`, while activity conversion uses a neutral shared contract. +- Frontend contract tests: **10 passed**. +- `npm run build`: **passed**; existing Vite chunk/dynamic-import warnings remain. +- `PYTHONPATH= uv run python -m compileall -q app tests`: **passed**. +- Diff security scan over backend code: no `shell=True`, `eval/exec`, pickle loading, or hardcoded-secret assignment introduced. +- `ruff`: unavailable; not counted as a pass. +- Browser viewport capture at 320×568/500×768: **not verified** because Chrome requested an OS remote-debugging permission; no permission dialog was clicked. + +## Risks / open boundaries + +- The bounded local path is deterministic E2E; it does not prove semantic LLM parity or production worker durability. +- Zep remains the default and compatibility runtime; consumer cutover is not complete. +- Dockerfile still runs the development stack; no production WSGI/worker/object-storage topology has been verified. +- `ruff` and real browser viewport QA remain blocked/unavailable. +- Fresh independent bounded reviewer `deleg_fb0f80b0`: **PASS** (`passed=true`, empty `security_concerns` and `logic_errors`) after current-code app/factory mismatch probes for create/get/update/list paths, pooled and NullPool SQLite FK probes, migration round-trip, and ORM/migration parity. Suggestions are non-blocking and recorded in `HANDOFF.md`. +- Fresh independent auxiliary reviewer `deleg_d128494e`: **PASS** (`passed=true`, empty `security_concerns` and `logic_errors`) after verifying all five routes, decorator ordering, safe errors, and 8/8 targeted tests. Wider SaaS/Zep/deployment review remains open. +- Current-tree bounded local import-boundary reviewer `deleg_65bce3de`: **PASS** (`passed=true`, empty `security_concerns` and `logic_errors`) after running the focused boundary command (**6 passed**) and isolated probes for five fresh local imports, shared `AgentActivity`, explicit Zep paths, the `graph_builder.Zep` seam, and subprocess isolation. Suggestions are non-blocking; this closes only the import-boundary slice, not semantic parity or full consumer cutover. + +## Next exact action + +Inspect the non-blocking reviewer suggestions only if they are useful for the next test-hardening pass. The bounded schema/TaskManager slice is approved; keep the broader SaaS/Zep/deployment blockers explicit in `HANDOFF.md` and continue consumer cutover. diff --git a/docs/engineering-log/2026-08-24-simulation-memory-fallback.md b/docs/engineering-log/2026-08-24-simulation-memory-fallback.md new file mode 100644 index 0000000..8941ec8 --- /dev/null +++ b/docs/engineering-log/2026-08-24-simulation-memory-fallback.md @@ -0,0 +1,53 @@ +# Simulation Memory Fallback Hardening — 2026-08-24 + +## Scope + +Close one local-memory safety gap without changing the migration decision: + +- `MEMORY_BACKEND=zep` remains the default compatibility path. +- `MEMORY_BACKEND=local` must use an explicitly injected, request-scoped entity-reader factory. +- Local mode must not silently construct `ZepEntityReader` when the factory is absent. + +## TDD evidence + +1. Added `test_local_backend_refuses_unscoped_zep_reader_fallback` to + `backend/tests/test_simulation_memory_injection.py`. +2. The test failed against the pre-change fallback behavior. +3. `SimulationManager.create_entity_reader()` now raises the stable internal + sentinel `local_entity_reader_factory_required` in local mode when no + factory is injected. +4. Explicit factory injection remains unchanged. + +## Verification + +- Local-memory regression slice: **11 passed**. +- Backend full suite: **73 passed**. +- Frontend contract tests: **10 passed**. +- Frontend production build: **PASS**; existing chunk-size/dynamic-import warnings remain. +- Python `compileall`: **PASS**. +- `git diff --check`: **PASS**. +- Target-slice added-lines security scan: **0 findings**. +- Full-diff scan matched two password patterns in test-only fixtures; no production credential was identified by that scan. +- `ruff` is unavailable in the current backend environment; no lint pass is claimed. + +## Independent review gate + +- Fresh read-only reviewer: `deleg_029828b6`. +- Scope was limited to `SimulationManager.create_entity_reader()` and + `test_local_backend_refuses_unscoped_zep_reader_fallback()`. +- Required five-key verdict: `passed=true`, `security_concerns=[]`, + `logic_errors=[]`. +- Reviewer-reported runtime spy: local mode made **0** Zep constructor calls; + the default path made **1** Zep constructor call. +- Reviewer suggestions are non-blocking: add explicit constructor-spy + regression coverage for the local no-call path and the legacy/default path. + They are deferred to a later test-hardening pass; no production behavior gap + was found in this bounded slice. +- This closes the review gate for this slice only. It is not approval of the + full migration, semantic parity, or production readiness. + +## Remaining migration boundary + +This is a fail-closed guard, not Zep removal or semantic parity proof. Remaining +work includes complete local consumer cutover, parity fixtures, durable resource +persistence, production workers/object storage, and the full tenant E2E matrix. diff --git a/docs/test-evidence/2026-08-23-baseline.md b/docs/test-evidence/2026-08-23-baseline.md new file mode 100644 index 0000000..2e5f5d8 --- /dev/null +++ b/docs/test-evidence/2026-08-23-baseline.md @@ -0,0 +1,27 @@ +# Baseline Test Evidence — 2026-08-23 + +**Repo:** `/Users/kunthawat/Gitea/MiroFish` +**HEAD:** `89d04e7` +**Purpose:** architecture-study baseline; no production implementation changes + +## Commands + +| Command | Result | Notes | +|---|---|---| +| `git status --short --branch` | PASS | clean at baseline; plan/log artifacts were added after study | +| `git diff --check` | PASS | no whitespace errors | +| `npm run build` | PASS | Vite build succeeded; chunk-size warning and dynamic/static import warning remain | +| `python3 -m compileall -q backend/app backend/run.py backend/scripts` | PASS | syntax-only; host Python is 3.14, project requires `<3.13` | + +## Not yet run + +- Backend runtime tests (no project test suite/runner configured for the application) +- Authentication/tenant isolation tests (not implemented) +- Zep replacement parity tests (not implemented) +- Thai/English UI visual/E2E tests +- Production Docker/worker/restart tests +- CJK-free build gate (currently expected to fail) + +## Verification rule + +A future milestone is not complete from code presence alone. Attach command output, relevant fixture counts, and role/locale/worker evidence here or in a dated evidence file before marking it complete. diff --git a/docs/test-evidence/2026-08-23-saas-foundation.md b/docs/test-evidence/2026-08-23-saas-foundation.md new file mode 100644 index 0000000..90052d0 --- /dev/null +++ b/docs/test-evidence/2026-08-23-saas-foundation.md @@ -0,0 +1,50 @@ +# SaaS Foundation Batch Evidence — 2026-08-23 + +**Branch:** `feat/saas-foundation` +**Scope:** locale/error foundation, identity schema/session, auth/admin API, project/graph/simulation/report/task scope, frontend login guard, report API contract, local memory schema/repository, strict LLM extraction contract +**Status:** in progress — reviewer gate blocked by delegation owner exits; simulation/report/task scope and login foundation are now implemented, but Zep replacement and durable SaaS storage are not complete + +## Verified commands + +| Command | Result | +|---|---| +| `PYTHONPATH=backend uv run --project backend --python 3.11 pytest backend/tests -q` | **55 passed** | +| `node --test frontend/tests/i18n-contract.test.mjs frontend/tests/api-error.test.mjs frontend/tests/report-api-contract.test.mjs frontend/tests/auth-contract.test.mjs` | **10 passed** | +| `npm run build` | **PASS**; existing chunk-size and pendingUpload import warnings remain | +| `python3.11 -m compileall -q backend/app backend/migrations` | **PASS** | +| `alembic upgrade head` on temporary SQLite database | **PASS**; revisions `0001_identity`, `0002_sessions` applied | +| `alembic check` on temporary SQLite database | **PASS**; no new upgrade operations | +| `git diff --cached --check` via subprocess | **PASS** | +| text-artifact CJK gate | **PASS**; binary icon excluded and visually checked as graphic-only | + +## Delivered production surfaces + +- `th/en` locale policy; legacy `zh` normalizes to Thai +- structured API errors and raw request-body redaction +- SQLAlchemy identity/organization/membership/session models +- Alembic migrations +- Argon2 password hashing +- HttpOnly session cookie + signed CSRF cookie/header check +- `/api/auth/login`, `/api/auth/me`, `/api/auth/logout` +- `/api/admin/users` GET/POST with role policy and redacted serializer +- tenant/owner fields and scoped lookup/list for projects +- auth + owner scope on graph project get/list/delete/reset and graph build/ontology entry points +- blueprint-wide auth + CSRF enforcement on simulation/report APIs +- fail-closed project/simulation/report/task scope checks for IDs in paths, query strings, and JSON bodies +- frontend auth store, `/login` route, protected workspace route guard, and localized login view +- report-status frontend client corrected to POST `{ task_id, simulation_id }` +- durable local memory tables (`memory_graphs`, `memory_episodes`, `memory_nodes`, `memory_edges`) and tenant-scoped SQLAlchemy repository +- strict Pydantic LLM extraction schema, bounded prompt builder, JSON parser, and deterministic extraction-to-repository service +- golden parity fixture for `LocalEntityReader` and `LocalMemoryTools.quick_search` against the legacy entity/search output shapes +- explicit `MEMORY_BACKEND=local` switch for simulation entity routes and preparation; default remains `zep` until migration completes +- `SimulationManager` entity-reader injection and worker-owned local-reader session lifecycle +- explicit CORS allowlist with credential support + +## Known incomplete surfaces + +- Existing product resources still use filesystem/in-memory persistence; no durable simulation/report/job tables yet. +- Legacy route handlers still return raw `str(e)` in some untouched modules. +- No admin user-management UI or super-admin settings UI yet. +- No durable worker queue or object-storage artifact layer. +- Zep dependency/runtime remains; local memory schema/contract exists, but GraphBuilder/ProfileGenerator/ReportAgent consumers have not been switched. +- Independent reviewer verdict is blocked: delegation owners exited before recording JSON results. diff --git a/docs/test-evidence/2026-08-24-saas-foundation-verification.md b/docs/test-evidence/2026-08-24-saas-foundation-verification.md new file mode 100644 index 0000000..f285a9b --- /dev/null +++ b/docs/test-evidence/2026-08-24-saas-foundation-verification.md @@ -0,0 +1,42 @@ +# SaaS Foundation Verification Evidence — 2026-08-24 + +**Repo:** `/Users/kunthawat/Gitea/MiroFish` +**Branch:** `feat/saas-foundation` +**Status:** verified bounded changes; broader migration remains in progress + +## Commands and results + +| Command | Result | +|---|---| +| `PYTHONPATH= uv run pytest -q` from `backend` after app/factory isolation, auxiliary/idempotency hardening, and local consumer boundary fixes | **123 passed**; four Alembic deprecation warnings | +| Schema regression (`tests/test_saas_operations_schema.py` + TaskManager isolation tests) | **16 passed** on the current tree; migration round-trip/model alignment, PRAGMA/FK/delete, cross-app binding, and background-use coverage | +| Auxiliary API security regression | **8 passed**; all five auxiliary routes require authentication, LLM mutations require CSRF/idempotency, safe errors are redacted, and both mutation replays are verified | +| Idempotency API regression | **5 passed**; same-route replay/conflict semantics, cross-route/body fingerprint separation, multipart content hashing, and repeated-file coverage are verified | +| Local memory/Zep import-boundary regression (`tests/test_local_service_import.py` + `tests/test_local_zep_boundary.py`) | **5 fresh-import tests passed** plus the activity-dict boundary regression; fresh subprocesses import the local updater, `SimulationManager`, `SimulationRunner`, `ReportAgent`, and `GraphBuilderService` without loading `zep_cloud` | +| `node --test tests/*.mjs` from `frontend` | **10 passed** | +| `npm run build` from `frontend` | **PASS**; Vite warns about `pendingUpload` dynamic/static imports and one chunk over 500 kB | +| `PYTHONPATH= uv run python -m compileall -q app tests` | **PASS** | +| Temporary SQLite Alembic upgrade → downgrade `0003_memory` → upgrade head → `alembic check` | **PASS** | +| Diff-only security scan over first-party backend runtime code | **0 hits** for `shell=True`, `eval/exec`, pickle loading, or hardcoded-secret assignment | +| `ruff --version` | **unavailable**; not a pass | +| Dev server `curl http://127.0.0.1:3000/login` | **PASS** | +| Independent bounded schema/TaskManager review `deleg_fb0f80b0` | **PASS**; reviewer snapshot ran 15 tests with `passed=true` and empty blocking arrays; the current tree rerun is **16 passed**. Current-code app/factory, SQLite FK, migration round-trip, and ORM/migration parity probes passed. Non-blocking suggestions recorded in `HANDOFF.md`; this is not full-system approval | +| Independent auxiliary API review `deleg_d128494e` | **PASS**; `passed=true`, empty `security_concerns` and `logic_errors`; current auxiliary security regression is **8 passed**. This is only the template/agent-group slice, not full-system approval | +| Independent current-tree local import-boundary review `deleg_65bce3de` | **PASS**; `passed=true`, empty blocking arrays; reviewer ran the focused boundary command (**6 passed**) and isolated probes for five fresh local imports, shared `AgentActivity`, explicit Zep paths, the `graph_builder.Zep` seam, and subprocess isolation. Its non-blocking suggestions (permanent regression guards for explicit Zep-path imports, `AgentActivity` identity, and the `graph_builder.Zep` seam) were incorporated; the focused boundary file now runs **9 passed** and the full backend suite is **127 passed**. This closes only the bounded import-boundary slice, not semantic parity or full consumer cutover | +| Real browser 320×568 and 500×768 capture | **not verified**; Chrome requested OS remote-debugging permission and no dialog was clicked | + +## Scope of the green evidence + +- Durable TaskManager binds to the current app/session factory, fails closed if an app context has no factory or a manager crosses app/factory boundaries, and supports SQL-level organization/owner predicates. +- Shared SQLite engine creation enables foreign keys; invalid org/owner references and `SET NULL`/`CASCADE` delete actions are covered. +- GraphBuilder async task metadata carries organization/session context. +- Local bounded flow reaches persisted report output without constructing a Zep adapter. +- Frontend active routes use Thai/English locale contracts; build artifacts contain no CJK code points. + +## Not proven by this evidence + +- Full Zep/local semantic parity or complete consumer cutover. +- Production WSGI/worker/object-storage deployment. +- Mobile visual QA. +- Complete idempotency coverage for every retryable mutation. +- Independent full-system approval; broad reviewer `deleg_949ab2f4` timed out, prior narrow reviewer `deleg_2435ad1e` returned `passed=false` before the operation-time guard remediation, and fresh reviewer `deleg_fb0f80b0` passed only the bounded schema/TaskManager slice. Wider SaaS/Zep/deployment work is still not approved. diff --git a/frontend/index.html b/frontend/index.html index 1e3e818..2552d3c 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -1,15 +1,24 @@ - + - + - + - - CrowdSight - 预测万物 + + CrowdSight - Predict Everything
diff --git a/frontend/src/App.vue b/frontend/src/App.vue index b7cd71c..1cef9af 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -15,7 +15,7 @@ } #app { - font-family: 'JetBrains Mono', 'Space Grotesk', 'Noto Sans SC', monospace; + font-family: 'JetBrains Mono', 'Space Grotesk', 'Noto Sans Thai', 'Sarabun', system-ui, sans-serif; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; color: #000000; diff --git a/frontend/src/api/error.js b/frontend/src/api/error.js new file mode 100644 index 0000000..acaec63 --- /dev/null +++ b/frontend/src/api/error.js @@ -0,0 +1,15 @@ +const CJK_PATTERN = /[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/ +const SENSITIVE_PATTERN = /traceback|api[_-]?key|secret|password|token\s*=|(?:\/Users\/|\/app\/|[A-Za-z]:\\\\)/i + +const isSafeMessage = value => { + if (typeof value !== 'string' || value.length === 0 || value.length > 240) return false + if (value.includes('\n') || value.includes('\r')) return false + if (CJK_PATTERN.test(value) || SENSITIVE_PATTERN.test(value)) return false + return true +} + +export const getSafeApiErrorMessage = (error, fallback = 'Request failed') => { + const data = error?.response?.data || error?.data || error || {} + if (!data?.error_code || !isSafeMessage(data.message)) return fallback + return data.message +} diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 29ad26d..8539ce4 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -1,5 +1,6 @@ import axios from 'axios' import i18n from '../i18n' +import { getSafeApiErrorMessage } from './error' // 创建axios实例 // VITE_API_BASE_URL: set to backend URL for cross-origin deployments @@ -7,16 +8,28 @@ import i18n from '../i18n' // Falls back to '' (empty) when using vite proxy (same-origin / local dev) const service = axios.create({ baseURL: import.meta.env.VITE_API_BASE_URL || '', + withCredentials: true, timeout: 300000, // 5分钟超时（本体生成可能需要较长时间） headers: { 'Content-Type': 'application/json' } }) -// 请求拦截器 +const readCookie = name => { + const prefix = `${name}=` + const match = document.cookie.split('; ').find(cookie => cookie.startsWith(prefix)) + return match ? decodeURIComponent(match.slice(prefix.length)) : '' +} + +// Request interceptor service.interceptors.request.use( config => { config.headers['Accept-Language'] = i18n.global.locale.value + const method = (config.method || 'get').toLowerCase() + if (!['get', 'head', 'options'].includes(method)) { + const csrfToken = readCookie('crowdsight_csrf') + if (csrfToken) config.headers['X-CSRF-Token'] = csrfToken + } return config }, error => { @@ -25,36 +38,31 @@ service.interceptors.request.use( } ) -// 响应拦截器（容错重试机制） +// Response interceptor service.interceptors.response.use( response => { const res = response.data - - // 如果返回的状态码不是success，则抛出错误 + const fallbackMessage = i18n.global.t('common.requestFailed') + if (!res.success && res.success !== undefined) { - console.error('API Error:', res.error || res.message || 'Unknown error') - return Promise.reject(new Error(res.error || res.message || 'Error')) + const safeMessage = getSafeApiErrorMessage({ data: res }, fallbackMessage) + console.error('API Error:', res.error_code || 'request_failed') + return Promise.reject(new Error(safeMessage)) } - + return res }, error => { - // Extract actual error message from response body - const backendError = error.response?.data?.error || error.response?.data?.message - const errorMsg = backendError || error.message || 'Unknown error' - console.error('Response error:', errorMsg, error) - - // 处理超时 - if (error.code === 'ECONNABORTED' && error.message.includes('timeout')) { - console.error('Request timeout') - } - - // 处理网络错误 - if (error.message === 'Network Error') { - console.error('Network error - please check your connection') - } - - // Reject with a meaningful error message + const timeout = error.code === 'ECONNABORTED' && error.message?.includes('timeout') + const network = error.message === 'Network Error' + const fallbackMessage = timeout + ? i18n.global.t('common.requestTimeout') + : network + ? i18n.global.t('common.networkError') + : i18n.global.t('common.requestFailed') + const errorMsg = getSafeApiErrorMessage(error, fallbackMessage) + + console.error('Response error:', error.response?.status || error.code || 'unknown') const enhancedError = new Error(errorMsg) enhancedError.response = error.response return Promise.reject(enhancedError) diff --git a/frontend/src/api/report.js b/frontend/src/api/report.js index c89a67d..353062a 100644 --- a/frontend/src/api/report.js +++ b/frontend/src/api/report.js @@ -9,11 +9,14 @@ export const generateReport = (data) => { } /** - * 获取报告生成状态 - * @param {string} reportId + * Get report generation status. + * @param {{taskId?: string, simulationId?: string}} identifiers */ -export const getReportStatus = (reportId) => { - return service.get(`/api/report/generate/status`, { params: { report_id: reportId } }) +export const getReportStatus = ({ taskId, simulationId } = {}) => { + const data = {} + if (taskId) data.task_id = taskId + if (simulationId) data.simulation_id = simulationId + return service.post('/api/report/generate/status', data) } /** diff --git a/frontend/src/components/HistoryDatabase.vue b/frontend/src/components/HistoryDatabase.vue index d6c6e9a..e0d1d18 100644 --- a/frontend/src/components/HistoryDatabase.vue +++ b/frontend/src/components/HistoryDatabase.vue @@ -445,7 +445,7 @@ const loadHistory = async () => { projects.value = response.data || [] } } catch (error) { - console.error('加载历史项目失败:', error) + console.error('Failed to load project history:', error) projects.value = [] } finally { loading.value = false diff --git a/frontend/src/components/Step1GraphBuild.vue b/frontend/src/components/Step1GraphBuild.vue index 687d1c7..d2e3579 100644 --- a/frontend/src/components/Step1GraphBuild.vue +++ b/frontend/src/components/Step1GraphBuild.vue @@ -213,7 +213,7 @@ const creatingSimulation = ref(false) // 进入环境搭建 - 创建 simulation 并跳转 const handleEnterEnvSetup = async () => { if (!props.projectData?.project_id || !props.projectData?.graph_id) { - console.error('缺少项目或图谱信息') + console.error('Project or graph information is missing') return } @@ -234,11 +234,11 @@ const handleEnterEnvSetup = async () => { params: { simulationId: res.data.simulation_id } }) } else { - console.error('创建模拟失败:', res.error) + console.error('Simulation creation failed:', res.error) alert(t('step1.createSimulationFailed', { error: res.error || t('common.unknownError') })) } } catch (err) { - console.error('创建模拟异常:', err) + console.error('Simulation creation error:', err) alert(t('step1.createSimulationException', { error: err.message })) } finally { creatingSimulation.value = false diff --git a/frontend/src/components/Step2EnvSetup.vue b/frontend/src/components/Step2EnvSetup.vue index 823dd6e..a8aea0b 100644 --- a/frontend/src/components/Step2EnvSetup.vue +++ b/frontend/src/components/Step2EnvSetup.vue @@ -757,16 +757,16 @@ const customMaxRounds = ref(40) // 默认推荐40轮 // Watch stage to update phase watch(currentStage, (newStage) => { - if (newStage === '生成Agent人设' || newStage === 'generating_profiles') { + if (newStage === 'generating_profiles' || newStage === 'generating_profiles') { phase.value = 1 - } else if (newStage === '生成模拟配置' || newStage === 'generating_config') { + } else if (newStage === 'generating_config' || newStage === 'generating_config') { phase.value = 2 // 进入配置生成阶段，开始轮询配置 if (!configTimer) { addLog(t('log.startGeneratingConfig')) startConfigPolling() } - } else if (newStage === '准备模拟脚本' || newStage === 'copying_scripts') { + } else if (newStage === 'copying_scripts' || newStage === 'copying_scripts') { phase.value = 2 // 仍属于配置阶段 } }) @@ -1039,7 +1039,7 @@ const pollPrepareStatus = async () => { } } } catch (err) { - console.warn('轮询状态失败:', err) + console.warn('Polling status failed:', err) } } @@ -1083,7 +1083,7 @@ const fetchProfilesRealtime = async () => { } } } catch (err) { - console.warn('获取 Profiles 失败:', err) + console.warn('Failed to fetch profiles:', err) } } @@ -1151,7 +1151,7 @@ const fetchConfigRealtime = async () => { } } } catch (err) { - console.warn('获取 Config 失败:', err) + console.warn('Failed to fetch config:', err) } } diff --git a/frontend/src/components/Step3Simulation.vue b/frontend/src/components/Step3Simulation.vue index 87a8c15..0bc9141 100644 --- a/frontend/src/components/Step3Simulation.vue +++ b/frontend/src/components/Step3Simulation.vue @@ -431,7 +431,7 @@ const doStartSimulation = async () => { startStatusPolling() startDetailPolling() } else { - startError.value = res.error || '启动失败' + startError.value = res.error || 'Start failed' addLog(t('log.startFailed', { error: res.error || t('common.unknownError') })) emit('update-status', 'error') } @@ -536,7 +536,7 @@ const fetchRunStatus = async () => { } } } catch (err) { - console.warn('获取运行状态失败:', err) + console.warn('Failed to fetch run status:', err) } } @@ -594,7 +594,7 @@ const fetchRunStatusDetail = async () => { // 新动作会在底部追加 } } catch (err) { - console.warn('获取详细状态失败:', err) + console.warn('Failed to fetch detailed status:', err) } } diff --git a/frontend/src/components/Step4Report.vue b/frontend/src/components/Step4Report.vue index 8e53ceb..19b0a08 100644 --- a/frontend/src/components/Step4Report.vue +++ b/frontend/src/components/Step4Report.vue @@ -553,31 +553,31 @@ const parseInsightForge = (text) => { } try { - // 提取分析问题 - const queryMatch = text.match(/分析问题:\s*(.+?)(?:\n|$)/) + // 提取分析Question + const queryMatch = text.match(/Analysis question:\s*(.+?)(?:\n|$)/) if (queryMatch) result.query = queryMatch[1].trim() // 提取预测场景 - const reqMatch = text.match(/预测场景:\s*(.+?)(?:\n|$)/) + const reqMatch = text.match(/Prediction scenario:\s*(.+?)(?:\n|$)/) if (reqMatch) result.simulationRequirement = reqMatch[1].trim() - // 提取统计数据 - 匹配"相关预测事实: X条"格式 - const factMatch = text.match(/相关预测事实:\s*(\d+)/) - const entityMatch = text.match(/涉及实体:\s*(\d+)/) - const relMatch = text.match(/关系链:\s*(\d+)/) + // 提取统计数据 - 匹配"Related facts: X条"格式 + const factMatch = text.match(/Related facts:\s*(\d+)/) + const entityMatch = text.match(/Entities:\s*(\d+)/) + const relMatch = text.match(/Relationship chains:\s*(\d+)/) if (factMatch) result.stats.facts = parseInt(factMatch[1]) if (entityMatch) result.stats.entities = parseInt(entityMatch[1]) if (relMatch) result.stats.relationships = parseInt(relMatch[1]) - // 提取子问题 - 完整提取，不限制数量 - const subQSection = text.match(/### 分析的子问题\n([\s\S]*?)(?=\n###|$)/) + // 提取子Question - 完整提取，不限制数量 + const subQSection = text.match(/### Sub-questions\n([\s\S]*?)(?=\n###|$)/) if (subQSection) { const lines = subQSection[1].split('\n').filter(l => l.match(/^\d+\./)) result.subQueries = lines.map(l => l.replace(/^\d+\.\s*/, '').trim()).filter(Boolean) } // 提取关键事实 - 完整提取，不限制数量 - const factsSection = text.match(/### 【关键事实】[\s\S]*?\n([\s\S]*?)(?=\n###|$)/) + const factsSection = text.match(/### Key facts[\s\S]*?\n([\s\S]*?)(?=\n###|$)/) if (factsSection) { const lines = factsSection[1].split('\n').filter(l => l.match(/^\d+\./)) result.facts = lines.map(l => { @@ -587,15 +587,15 @@ const parseInsightForge = (text) => { } // 提取核心实体 - 完整提取，包含摘要和相关事实数 - const entitySection = text.match(/### 【核心实体】\n([\s\S]*?)(?=\n###|$)/) + const entitySection = text.match(/### Core entities\n([\s\S]*?)(?=\n###|$)/) if (entitySection) { const entityText = entitySection[1] // 按 "- **" 分割实体块 const entityBlocks = entityText.split(/\n(?=- \*\*)/).filter(b => b.trim().startsWith('- **')) result.entities = entityBlocks.map(block => { const nameMatch = block.match(/^-\s*\*\*(.+?)\*\*\s*\((.+?)\)/) - const summaryMatch = block.match(/摘要:\s*"?(.+?)"?(?:\n|$)/) - const relatedMatch = block.match(/相关事实:\s*(\d+)/) + const summaryMatch = block.match(/Summary:\s*"?(.+?)"?(?:\n|$)/) + const relatedMatch = block.match(/Related facts:\s*(\d+)/) return { name: nameMatch ? nameMatch[1].trim() : '', type: nameMatch ? nameMatch[2].trim() : '', @@ -606,7 +606,7 @@ const parseInsightForge = (text) => { } // 提取关系链 - 完整提取，不限制数量 - const relSection = text.match(/### 【关系链】\n([\s\S]*?)(?=\n###|$)/) + const relSection = text.match(/### Relationship chains\n([\s\S]*?)(?=\n###|$)/) if (relSection) { const lines = relSection[1].split('\n').filter(l => l.trim().startsWith('-')) result.relations = lines.map(l => { @@ -635,21 +635,21 @@ const parsePanorama = (text) => { try { // 提取查询 - const queryMatch = text.match(/查询:\s*(.+?)(?:\n|$)/) + const queryMatch = text.match(/Query:\s*(.+?)(?:\n|$)/) if (queryMatch) result.query = queryMatch[1].trim() // 提取统计数据 - const nodesMatch = text.match(/总节点数:\s*(\d+)/) - const edgesMatch = text.match(/总边数:\s*(\d+)/) - const activeMatch = text.match(/当前有效事实:\s*(\d+)/) - const histMatch = text.match(/历史\/过期事实:\s*(\d+)/) + const nodesMatch = text.match(/Total nodes:\s*(\d+)/) + const edgesMatch = text.match(/Total edges:\s*(\d+)/) + const activeMatch = text.match(/Active facts:\s*(\d+)/) + const histMatch = text.match(/Historical facts:\s*(\d+)/) if (nodesMatch) result.stats.nodes = parseInt(nodesMatch[1]) if (edgesMatch) result.stats.edges = parseInt(edgesMatch[1]) if (activeMatch) result.stats.activeFacts = parseInt(activeMatch[1]) if (histMatch) result.stats.historicalFacts = parseInt(histMatch[1]) // 提取当前有效事实 - 完整提取，不限制数量 - const activeSection = text.match(/### 【当前有效事实】[\s\S]*?\n([\s\S]*?)(?=\n###|$)/) + const activeSection = text.match(/### Active facts[\s\S]*?\n([\s\S]*?)(?=\n###|$)/) if (activeSection) { const lines = activeSection[1].split('\n').filter(l => l.match(/^\d+\./)) result.activeFacts = lines.map(l => { @@ -660,7 +660,7 @@ const parsePanorama = (text) => { } // 提取历史/过期事实 - 完整提取，不限制数量 - const histSection = text.match(/### 【历史\/过期事实】[\s\S]*?\n([\s\S]*?)(?=\n###|$)/) + const histSection = text.match(/### Historical facts[\s\S]*?\n([\s\S]*?)(?=\n###|$)/) if (histSection) { const lines = histSection[1].split('\n').filter(l => l.match(/^\d+\./)) result.historicalFacts = lines.map(l => { @@ -670,7 +670,7 @@ const parsePanorama = (text) => { } // 提取涉及实体 - 完整提取，不限制数量 - const entitySection = text.match(/### 【涉及实体】\n([\s\S]*?)(?=\n###|$)/) + const entitySection = text.match(/### Entities\n([\s\S]*?)(?=\n###|$)/) if (entitySection) { const lines = entitySection[1].split('\n').filter(l => l.trim().startsWith('-')) result.entities = lines.map(l => { @@ -699,24 +699,24 @@ const parseInterview = (text) => { try { // 提取采访主题 - const topicMatch = text.match(/\*\*采访主题:\*\*\s*(.+?)(?:\n|$)/) + const topicMatch = text.match(/\*\*Interview topic:\*\*\s*(.+?)(?:\n|$)/) if (topicMatch) result.topic = topicMatch[1].trim() // 提取采访人数（如 "5 / 9 位模拟Agent"） - const countMatch = text.match(/\*\*采访人数:\*\*\s*(\d+)\s*\/\s*(\d+)/) + const countMatch = text.match(/\*\*Interviewed:\*\*\s*(\d+)\s*\/\s*(\d+)/) if (countMatch) { result.successCount = parseInt(countMatch[1]) result.totalCount = parseInt(countMatch[2]) result.agentCount = `${countMatch[1]} / ${countMatch[2]}` } - // 提取采访对象选择理由 - const reasonMatch = text.match(/### 采访对象选择理由\n([\s\S]*?)(?=\n---\n|\n### 采访实录)/) + // 提取Interviewee selection reasoning + const reasonMatch = text.match(/### Interviewee selection reasoning\n([\s\S]*?)(?=\n---\n|\n### Interview transcripts)/) if (reasonMatch) { result.selectionReason = reasonMatch[1].trim() } - // 解析每个人的选择理由 + // 解析每个人的Selection reason const parseIndividualReasons = (reasonText) => { const reasons = {} if (!reasonText) return reasons @@ -738,10 +738,10 @@ const parseInterview = (text) => { reasonStart = headerMatch[2] } - // 格式2: - 选择名字（index X）：理由 - // 例如: - 选择家长_601（index 0）：作为家长群体代表... + // 格式2: - Select名字（index X）：理由 + // 例如: - Select家长_601（index 0）：作为家长群体代表... if (!headerMatch) { - headerMatch = line.match(/^-\s*选择([^（(]+)(?:[（(]index\s*=?\s*\d+[)）])?[：:]\s*(.*)/) + headerMatch = line.match(/^-\s*Select([^（(]+)(?:[（(]index\s*=?\s*\d+[)）])?[：:]\s*(.*)/) if (headerMatch) { name = headerMatch[1].trim() reasonStart = headerMatch[2] @@ -766,7 +766,7 @@ const parseInterview = (text) => { // 开始新的人 currentName = name currentReason = reasonStart ? [reasonStart.trim()] : [] - } else if (currentName && line.trim() && !line.match(/^未选|^综上|^最终选择/)) { + } else if (currentName && line.trim() && !line.match(/^Not selected|^In summary|^Final selection/)) { // 理由的续行（排除结尾总结段落） currentReason.push(line.trim()) } @@ -783,7 +783,7 @@ const parseInterview = (text) => { const individualReasons = parseIndividualReasons(result.selectionReason) // 提取每个采访记录 - const interviewBlocks = text.split(/#### 采访 #\d+:/).slice(1) + const interviewBlocks = text.split(/#### Interview #\d+:/).slice(1) interviewBlocks.forEach((block, index) => { const interview = { @@ -808,24 +808,24 @@ const parseInterview = (text) => { if (nameRoleMatch) { interview.name = nameRoleMatch[1].trim() interview.role = nameRoleMatch[2].trim() - // 设置该人的选择理由 + // 设置该人的Selection reason interview.selectionReason = individualReasons[interview.name] || '' } // 提取简介 - const bioMatch = block.match(/_简介:\s*([\s\S]*?)_\n/) + const bioMatch = block.match(/_Bio:\s*([\s\S]*?)_\n/) if (bioMatch) { interview.bio = bioMatch[1].trim().replace(/\.\.\.$/, '...') } - // 提取问题列表 + // 提取Question列表 const qMatch = block.match(/\*\*Q:\*\*\s*([\s\S]*?)(?=\n\n\*\*A:\*\*|\*\*A:\*\*)/) if (qMatch) { const qText = qMatch[1].trim() - // 按数字编号分割问题 + // 按数字编号分割Question const questions = qText.split(/\n\d+\.\s+/).filter(q => q.trim()) if (questions.length > 0) { - // 如果第一个问题前面有"1."，需要特殊处理 + // 如果第一个Question前面有"1."，需要特殊处理 const firstQ = qText.match(/^1\.\s+(.+)/) if (firstQ) { interview.questions = [firstQ[1].trim(), ...questions.slice(1).map(q => q.trim())] @@ -836,13 +836,13 @@ const parseInterview = (text) => { } // 提取回答 - 分Twitter和Reddit - const answerMatch = block.match(/\*\*A:\*\*\s*([\s\S]*?)(?=\*\*关键引言|$)/) + const answerMatch = block.match(/\*\*A:\*\*\s*([\s\S]*?)(?=\*\*Key quotes|$)/) if (answerMatch) { const answerText = answerMatch[1].trim() // 分离Twitter和Reddit回答 - const twitterMatch = answerText.match(/【Twitter平台回答】\n?([\s\S]*?)(?=【Reddit平台回答】|$)/) - const redditMatch = answerText.match(/【Reddit平台回答】\n?([\s\S]*?)$/) + const twitterMatch = answerText.match(/[Twitter response]\n?([\s\S]*?)(?=[Reddit response]|$)/) + const redditMatch = answerText.match(/[Reddit response]\n?([\s\S]*?)$/) if (twitterMatch) { interview.twitterAnswer = twitterMatch[1].trim() @@ -854,11 +854,11 @@ const parseInterview = (text) => { // 平台回退逻辑（兼容旧格式：只有一个平台标记的情况） if (!twitterMatch && redditMatch) { // 只有 Reddit 回答，仅在非占位文本时复制为默认显示 - if (interview.redditAnswer && interview.redditAnswer !== '（该平台未获得回复）') { + if (interview.redditAnswer && interview.redditAnswer !== '(No response from this platform)') { interview.twitterAnswer = interview.redditAnswer } } else if (twitterMatch && !redditMatch) { - if (interview.twitterAnswer && interview.twitterAnswer !== '（该平台未获得回复）') { + if (interview.twitterAnswer && interview.twitterAnswer !== '(No response from this platform)') { interview.redditAnswer = interview.twitterAnswer } } else if (!twitterMatch && !redditMatch) { @@ -867,8 +867,8 @@ const parseInterview = (text) => { } } - // 提取关键引言（兼容多种引号格式） - const quotesMatch = block.match(/\*\*关键引言:\*\*\n([\s\S]*?)(?=\n---|\n####|$)/) + // 提取Key quotes（兼容多种引号格式） + const quotesMatch = block.match(/\*\*Key quotes:\*\*\n([\s\S]*?)(?=\n---|\n####|$)/) if (quotesMatch) { const quotesText = quotesMatch[1] // 优先匹配 > "text" 格式 @@ -890,7 +890,7 @@ const parseInterview = (text) => { }) // 提取采访摘要 - const summaryMatch = text.match(/### 采访摘要与核心观点\n([\s\S]*?)$/) + const summaryMatch = text.match(/### Interview summary and key perspectives\n([\s\S]*?)$/) if (summaryMatch) { result.summary = summaryMatch[1].trim() } @@ -912,22 +912,22 @@ const parseQuickSearch = (text) => { try { // 提取搜索查询 - const queryMatch = text.match(/搜索查询:\s*(.+?)(?:\n|$)/) + const queryMatch = text.match(/Search query:\s*(.+?)(?:\n|$)/) if (queryMatch) result.query = queryMatch[1].trim() // 提取结果数量 - const countMatch = text.match(/找到\s*(\d+)\s*条/) + const countMatch = text.match(/Found\s*(\d+)\s*results/) if (countMatch) result.count = parseInt(countMatch[1]) // 提取相关事实 - 完整提取，不限制数量 - const factsSection = text.match(/### 相关事实:\n([\s\S]*)$/) + const factsSection = text.match(/### Related facts:\n([\s\S]*)$/) if (factsSection) { const lines = factsSection[1].split('\n').filter(l => l.match(/^\d+\./)) result.facts = lines.map(l => l.replace(/^\d+\.\s*/, '').trim()).filter(Boolean) } // 尝试提取边信息（如果有） - const edgesSection = text.match(/### 相关边:\n([\s\S]*?)(?=\n###|$)/) + const edgesSection = text.match(/### Related edges:\n([\s\S]*?)(?=\n###|$)/) if (edgesSection) { const lines = edgesSection[1].split('\n').filter(l => l.trim().startsWith('-')) result.edges = lines.map(l => { @@ -940,7 +940,7 @@ const parseQuickSearch = (text) => { } // 尝试提取节点信息（如果有） - const nodesSection = text.match(/### 相关节点:\n([\s\S]*?)(?=\n###|$)/) + const nodesSection = text.match(/### Related nodes:\n([\s\S]*?)(?=\n###|$)/) if (nodesSection) { const lines = nodesSection[1].split('\n').filter(l => l.trim().startsWith('-')) result.nodes = lines.map(l => { @@ -1296,16 +1296,16 @@ const InterviewDisplay = { const activeIndex = ref(0) const expandedAnswers = ref(new Set()) - // 为每个问题-回答对维护独立的平台选择状态 + // 为每个Question-回答对维护独立的平台Select状态 const platformTabs = reactive({}) // { 'agentIdx-qIdx': 'twitter' | 'reddit' } - // 获取某个问题的当前平台选择 + // 获取某个Question的当前平台Select const getPlatformTab = (agentIdx, qIdx) => { const key = `${agentIdx}-${qIdx}` return platformTabs[key] || 'twitter' } - // 设置某个问题的平台选择 + // 设置某个Question的平台Select const setPlatformTab = (agentIdx, qIdx, platform) => { const key = `${agentIdx}-${qIdx}` platformTabs[key] = platform @@ -1331,22 +1331,22 @@ const InterviewDisplay = { const isPlaceholderText = (text) => { if (!text) return true const t = text.trim() - return t === '（该平台未获得回复）' || t === '(该平台未获得回复)' || t === '[无回复]' + return t === '(No response from this platform)' || t === '(No response from this platform)' || t === '[No response]' } - // 尝试按问题编号分割回答 + // 尝试按Question编号分割回答 const splitAnswerByQuestions = (answerText, questionCount) => { if (!answerText || questionCount <= 0) return [answerText] if (isPlaceholderText(answerText)) return [''] // 支持两种编号格式： - // 1. "问题X：" 或 "问题X:" （中文格式，后端新格式） + // 1. "QuestionX：" 或 "QuestionX:" （中文格式，后端新格式） // 2. "1. " 或 "\n1. " （数字+点，旧格式兼容） let matches = [] let match - // 优先尝试 "问题X：" 格式 - const cnPattern = /(?:^|[\r\n]+)问题(\d+)[：:]\s*/g + // 优先尝试 "QuestionX：" 格式 + const cnPattern = /(?:^|[\r\n]+)Question(\d+)[：:]\s*/g while ((match = cnPattern.exec(answerText)) !== null) { matches.push({ num: parseInt(match[1]), @@ -1370,7 +1370,7 @@ const InterviewDisplay = { // 如果没有找到编号或只找到一个，返回整体 if (matches.length <= 1) { const cleaned = answerText - .replace(/^问题\d+[：:]\s*/, '') + .replace(/^Question\d+[：:]\s*/, '') .replace(/^\d+\.\s+/, '') .trim() return [cleaned || answerText] @@ -1397,7 +1397,7 @@ const InterviewDisplay = { return [answerText] } - // 获取某个问题对应的回答 + // 获取某个Question对应的回答 const getAnswerForQuestion = (interview, qIdx, platform) => { const answer = platform === 'twitter' ? interview.twitterAnswer : (interview.redditAnswer || interview.twitterAnswer) if (!answer || isPlaceholderText(answer)) return answer || '' @@ -1410,11 +1410,11 @@ const InterviewDisplay = { return answers[qIdx] || '' } - // 分割失败：第一个问题返回完整回答，其余返回空 + // 分割失败：第一个Question返回完整回答，其余返回空 return qIdx === 0 ? answer : '' } - // 检查某个问题是否有双平台回答（过滤占位文本） + // 检查某个Question是否有双平台回答（过滤占位文本） const hasMultiplePlatforms = (interview, qIdx) => { if (!interview.twitterAnswer || !interview.redditAnswer) return false const twitterAnswer = getAnswerForQuestion(interview, qIdx, 'twitter') @@ -1469,9 +1469,9 @@ const InterviewDisplay = { ]) ]), - // Selection Reason - 选择理由 + // Selection Reason - Selection reason props.result.interviews[activeIndex.value]?.selectionReason && h('div', { class: 'selection-reason' }, [ - h('div', { class: 'reason-label' }, '选择理由'), + h('div', { class: 'reason-label' }, 'Selection reason'), h('div', { class: 'reason-content' }, props.result.interviews[activeIndex.value].selectionReason) ]), @@ -1781,7 +1781,7 @@ const activeStep = computed(() => { if (doneSteps.length > 0) return doneSteps[doneSteps.length - 1] // 否则返回第一个步骤 - return steps[0] || { noLabel: '--', title: '等待开始', status: 'todo', meta: '' } + return steps[0] || { noLabel: '--', title: 'Waiting to start', status: 'todo', meta: '' } }) const workflowSteps = computed(() => { @@ -2011,8 +2011,8 @@ const getActionLabel = (action) => { } const getLogLevelClass = (log) => { - if (log.includes('ERROR') || log.includes('错误')) return 'error' - if (log.includes('WARNING') || log.includes('警告')) return 'warning' + if (log.includes('ERROR') || log.includes('ERROR')) return 'error' + if (log.includes('WARNING') || log.includes('WARNING')) return 'warning' // INFO 使用默认颜色，不标记为 success return '' } @@ -2103,7 +2103,7 @@ const extractFinalContent = (response) => { } // 尝试找 最终答案: 后面的内容 - const chineseFinalMatch = response.match(/最终答案[:：]\s*\n*([\s\S]*)$/i) + const chineseFinalMatch = response.match(/Final Answer:\s*\n*([\s\S]*)$/i) if (chineseFinalMatch) { return chineseFinalMatch[1].trim() } @@ -3903,7 +3903,7 @@ watch(() => props.reportId, (newId) => { overflow: hidden; } -/* Selection Reason - 选择理由 */ +/* Selection Reason - Selection reason */ :deep(.interview-display .selection-reason) { background: #F8FAFC; border: 1px solid #E2E8F0; diff --git a/frontend/src/components/Step5Interaction.vue b/frontend/src/components/Step5Interaction.vue index 44042dc..f751483 100644 --- a/frontend/src/components/Step5Interaction.vue +++ b/frontend/src/components/Step5Interaction.vue @@ -728,9 +728,9 @@ const sendToAgent = async (message) => { const historyContext = chatHistory.value .filter(msg => msg.content !== message) .slice(-6) - .map(msg => `${msg.role === 'user' ? '提问者' : '你'}：${msg.content}`) + .map(msg => `${msg.role === 'user' ? 'User' : 'Assistant'}: ${msg.content}`) .join('\n') - prompt = `以下是我们之前的对话：\n${historyContext}\n\n现在我的新问题是：${message}` + prompt = `Here is our previous conversation:\n${historyContext}\n\nMy new question is:\n${message}` } const res = await interviewAgents({ diff --git a/frontend/src/i18n/index.js b/frontend/src/i18n/index.js index aa26553..89c22ad 100644 --- a/frontend/src/i18n/index.js +++ b/frontend/src/i18n/index.js @@ -1,6 +1,15 @@ import { createI18n } from 'vue-i18n' import languages from '../../../locales/languages.json' +export const DEFAULT_LOCALE = 'th' +export const SUPPORTED_LOCALES = ['th', 'en'] + +export const normalizeLocale = value => { + if (typeof value !== 'string') return DEFAULT_LOCALE + const baseLocale = value.trim().replace('_', '-').toLowerCase().split('-')[0] + return SUPPORTED_LOCALES.includes(baseLocale) ? baseLocale : DEFAULT_LOCALE +} + const localeFiles = import.meta.glob('../../../locales/!(languages).json', { eager: true }) const messages = {} @@ -8,18 +17,32 @@ const availableLocales = [] for (const path in localeFiles) { const key = path.match(/\/([^/]+)\.json$/)[1] - if (languages[key]) { + if (SUPPORTED_LOCALES.includes(key) && languages[key]) { messages[key] = localeFiles[path].default availableLocales.push({ key, label: languages[key].label }) } } -const savedLocale = localStorage.getItem('locale') || 'zh' +let savedLocale = null +try { + savedLocale = localStorage.getItem('locale') +} catch { + savedLocale = null +} + +const locale = normalizeLocale(savedLocale) +try { + localStorage.setItem('locale', locale) +} catch { + // Storage can be unavailable in private browsing; runtime locale still works. +} + +document.documentElement.lang = locale const i18n = createI18n({ legacy: false, - locale: savedLocale, - fallbackLocale: 'zh', + locale, + fallbackLocale: DEFAULT_LOCALE, messages }) diff --git a/frontend/src/router/index.js b/frontend/src/router/index.js index 62d2320..758c55f 100644 --- a/frontend/src/router/index.js +++ b/frontend/src/router/index.js @@ -1,52 +1,115 @@ import { createRouter, createWebHistory } from 'vue-router' import Home from '../views/Home.vue' +import LoginView from '../views/LoginView.vue' +import AdminView from '../views/AdminView.vue' +import SettingsView from '../views/SettingsView.vue' import Process from '../views/MainView.vue' import SimulationView from '../views/SimulationView.vue' import SimulationRunView from '../views/SimulationRunView.vue' import ReportView from '../views/ReportView.vue' import InteractionView from '../views/InteractionView.vue' +import { useAuth } from '../stores/auth' const routes = [ + { + path: '/login', + name: 'Login', + component: LoginView, + meta: { public: true }, + }, + { + path: '/admin', + name: 'Admin', + component: AdminView, + meta: { requiresAuth: true, adminOnly: true }, + }, + { + path: '/admin/settings', + name: 'AdminSettings', + component: SettingsView, + meta: { requiresAuth: true, adminOnly: true, superAdminOnly: true }, + }, { path: '/', name: 'Home', - component: Home + component: Home, + meta: { requiresAuth: true }, }, { path: '/process/:projectId', name: 'Process', component: Process, - props: true + props: true, + meta: { requiresAuth: true }, }, { path: '/simulation/:simulationId', name: 'Simulation', component: SimulationView, - props: true + props: true, + meta: { requiresAuth: true }, }, { path: '/simulation/:simulationId/start', name: 'SimulationRun', component: SimulationRunView, - props: true + props: true, + meta: { requiresAuth: true }, }, { path: '/report/:reportId', name: 'Report', component: ReportView, - props: true + props: true, + meta: { requiresAuth: true }, }, { path: '/interaction/:reportId', name: 'Interaction', component: InteractionView, - props: true - } + props: true, + meta: { requiresAuth: true }, + }, ] const router = createRouter({ history: createWebHistory(), - routes + routes, +}) + +router.beforeEach(async to => { + if (to.meta.public) return true + + const auth = useAuth() + const authenticated = await auth.load() + if (to.meta.requiresAuth && !authenticated) { + return { + name: 'Login', + query: { redirect: to.fullPath }, + } + } + + // Restrict admin-only views to admin / super_admin roles. + if (to.meta.adminOnly) { + const ident = auth.identity + const role = + ident?.data?.role ?? ident?.role ?? ident?.membership?.role ?? ident?.organization_membership?.role ?? '' + if (role !== 'admin' && role !== 'super_admin') { + return { name: 'Home' } + } + } + + // Restrict super-admin-only views to the super_admin role. + if (to.meta.superAdminOnly) { + const ident = auth.identity + const role = + ident?.data?.role ?? ident?.role ?? ident?.membership?.role ?? ident?.organization_membership?.role ?? '' + if (role !== 'super_admin') { + return { name: 'Admin' } + } + } + + return true }) export default router diff --git a/frontend/src/stores/auth.js b/frontend/src/stores/auth.js new file mode 100644 index 0000000..6512035 --- /dev/null +++ b/frontend/src/stores/auth.js @@ -0,0 +1,54 @@ +import { computed, ref } from 'vue' +import service from '../api' + +const identity = ref(null) +const initialized = ref(false) +const loading = ref(false) + +const authStore = { + identity, + initialized, + loading, + isAuthenticated: computed(() => Boolean(identity.value)), + + async load() { + if (initialized.value) return Boolean(identity.value) + loading.value = true + try { + const response = await service.get('/api/auth/me') + identity.value = response.data || null + } catch { + identity.value = null + } finally { + initialized.value = true + loading.value = false + } + return Boolean(identity.value) + }, + + async login(email, password, organizationSlug = '') { + loading.value = true + try { + const payload = { email, password } + if (organizationSlug) payload.organization_slug = organizationSlug + const response = await service.post('/api/auth/login', payload) + identity.value = response.data || null + initialized.value = true + return identity.value + } finally { + loading.value = false + } + }, + + async logout() { + try { + await service.post('/api/auth/logout') + } finally { + identity.value = null + initialized.value = true + } + }, +} + +export const useAuth = () => authStore +export default authStore diff --git a/frontend/src/views/AdminView.vue b/frontend/src/views/AdminView.vue new file mode 100644 index 0000000..4148f5d --- /dev/null +++ b/frontend/src/views/AdminView.vue @@ -0,0 +1,161 @@ + + + + + diff --git a/frontend/src/views/LoginView.vue b/frontend/src/views/LoginView.vue new file mode 100644 index 0000000..f67392c --- /dev/null +++ b/frontend/src/views/LoginView.vue @@ -0,0 +1,168 @@ + + + + + diff --git a/frontend/src/views/Process.vue b/frontend/src/views/Process.vue deleted file mode 100644 index d232f5c..0000000 --- a/frontend/src/views/Process.vue +++ /dev/null @@ -1,2068 +0,0 @@ - - - - - \ No newline at end of file diff --git a/frontend/src/views/SettingsView.vue b/frontend/src/views/SettingsView.vue new file mode 100644 index 0000000..4e44fbe --- /dev/null +++ b/frontend/src/views/SettingsView.vue @@ -0,0 +1,153 @@ + + + + + diff --git a/frontend/src/views/SimulationView.vue b/frontend/src/views/SimulationView.vue index 0101ad6..bc693bd 100644 --- a/frontend/src/views/SimulationView.vue +++ b/frontend/src/views/SimulationView.vue @@ -227,7 +227,7 @@ const checkAndStopRunningSimulation = async () => { } } catch (err) { // 检查环境状态失败不影响后续流程 - console.warn('检查模拟状态失败:', err) + console.warn('Failed to check simulation status:', err) } } diff --git a/frontend/tests/api-error.test.mjs b/frontend/tests/api-error.test.mjs new file mode 100644 index 0000000..2c52f24 --- /dev/null +++ b/frontend/tests/api-error.test.mjs @@ -0,0 +1,43 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +const load = async () => import('../src/api/error.js') + +test('accepts a bounded structured API message', async () => { + const { getSafeApiErrorMessage } = await load() + assert.equal( + getSafeApiErrorMessage( + { response: { data: { error_code: 'project_not_found', message: 'Project not found' } } }, + 'Request failed', + ), + 'Project not found', + ) +}) + +test('rejects raw traceback and filesystem leakage', async () => { + const { getSafeApiErrorMessage } = await load() + const error = { + response: { + data: { + error_code: 'internal_error', + message: 'Traceback: File "/Users/secret/app.py" API_KEY=sk-live-secret', + }, + }, + } + assert.equal(getSafeApiErrorMessage(error, 'Request failed'), 'Request failed') +}) + +test('rejects unstructured Chinese or oversized server text', async () => { + const { getSafeApiErrorMessage } = await load() + assert.equal( + getSafeApiErrorMessage({ response: { data: { error: '项目不存在' } } }, 'Request failed'), + 'Request failed', + ) + assert.equal( + getSafeApiErrorMessage( + { response: { data: { error_code: 'bad', message: 'x'.repeat(241) } } }, + 'Request failed', + ), + 'Request failed', + ) +}) diff --git a/frontend/tests/auth-contract.test.mjs b/frontend/tests/auth-contract.test.mjs new file mode 100644 index 0000000..1f888a2 --- /dev/null +++ b/frontend/tests/auth-contract.test.mjs @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import path from 'node:path' +import test from 'node:test' + +const repoRoot = path.resolve(import.meta.dirname, '../..') +const read = relative => fs.readFileSync(path.join(repoRoot, relative), 'utf8') + +test('frontend auth foundation exposes a public login route and guarded workspace', () => { + const router = read('frontend/src/router/index.js') + const authStore = read('frontend/src/stores/auth.js') + const loginView = read('frontend/src/views/LoginView.vue') + + assert.match(router, /path:\s*['"]\/login['"]/) + assert.match(router, /name:\s*['"]Login['"]/) + assert.match(router, /beforeEach/) + assert.match(router, /requiresAuth/) + assert.match(authStore, /api\/auth\/login|auth\/login/) + assert.doesNotMatch(authStore, /localStorage\.(setItem|save).*token/i) + assert.match(loginView, /auth\.login|login\(/) + assert.match(loginView, /name:\s*['"]Login['"]|router\.(push|replace)/) +}) diff --git a/frontend/tests/i18n-contract.test.mjs b/frontend/tests/i18n-contract.test.mjs new file mode 100644 index 0000000..d78e1d0 --- /dev/null +++ b/frontend/tests/i18n-contract.test.mjs @@ -0,0 +1,71 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import path from 'node:path' +import test from 'node:test' + +const repoRoot = path.resolve(import.meta.dirname, '../..') +const read = relativePath => fs.readFileSync(path.join(repoRoot, relativePath), 'utf8') +const parseJson = relativePath => JSON.parse(read(relativePath)) + +const flattenKeys = (value, prefix = '') => { + const keys = [] + for (const [key, child] of Object.entries(value)) { + const fullKey = prefix ? `${prefix}.${key}` : key + if (child && typeof child === 'object' && !Array.isArray(child)) { + keys.push(...flattenKeys(child, fullKey)) + } else { + keys.push(fullKey) + } + } + return keys.sort() +} + +test('locale registry exposes only Thai and English', () => { + const languages = parseJson('locales/languages.json') + assert.deepEqual(Object.keys(languages).sort(), ['en', 'th']) + assert.equal(languages.en.label, 'English') + assert.equal(languages.th.label, 'ไทย') +}) + +test('Thai and English translation dictionaries have the same keys', () => { + assert.deepEqual( + flattenKeys(parseJson('locales/en.json')), + flattenKeys(parseJson('locales/th.json')), + ) +}) + +test('frontend i18n uses Thai as the safe default and fallback', () => { + const source = read('frontend/src/i18n/index.js') + assert.match(source, /DEFAULT_LOCALE\s*=\s*['"]th['"]/) + assert.match(source, /fallbackLocale:\s*DEFAULT_LOCALE/) + assert.doesNotMatch(source, /\|\|\s*['"]zh['"]|fallbackLocale:\s*['"]zh['"]/) +}) + +test('HTML metadata has no Chinese locale or font dependency', () => { + const html = read('frontend/index.html') + assert.match(html, / { + const distRoot = path.join(repoRoot, 'frontend/dist') + const binaryExtensions = new Set(['.png', '.jpg', '.jpeg', '.gif', '.webp', '.ico']) + const textFiles = [] + + const visit = directory => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const entryPath = path.join(directory, entry.name) + if (entry.isDirectory()) { + visit(entryPath) + } else if (!binaryExtensions.has(path.extname(entry.name).toLowerCase())) { + textFiles.push(entryPath) + } + } + } + + visit(distRoot) + for (const filePath of textFiles) { + const content = fs.readFileSync(filePath, 'utf8') + assert.doesNotMatch(content, /[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/, filePath) + } +}) diff --git a/frontend/tests/report-api-contract.test.mjs b/frontend/tests/report-api-contract.test.mjs new file mode 100644 index 0000000..48be224 --- /dev/null +++ b/frontend/tests/report-api-contract.test.mjs @@ -0,0 +1,18 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import path from 'node:path' +import test from 'node:test' + +const repoRoot = path.resolve(import.meta.dirname, '../..') +const reportApi = fs.readFileSync( + path.join(repoRoot, 'frontend/src/api/report.js'), + 'utf8', +) + +test('report status client matches backend POST body contract', () => { + assert.ok(reportApi.includes("service.post('/api/report/generate/status', data)")) + assert.match(reportApi, /task_id/) + assert.match(reportApi, /simulation_id/) + assert.doesNotMatch(reportApi, /service\.get\([^\n]*generate\/status/) + assert.doesNotMatch(reportApi, /params:\s*\{\s*report_id/) +}) diff --git a/locales/en.json b/locales/en.json index ead53bb..b1d9633 100644 --- a/locales/en.json +++ b/locales/en.json @@ -13,6 +13,10 @@ "failed": "Failed", "unknown": "Unknown", "unknownError": "Unknown error", + "requestFailed": "Request failed. Please try again.", + "networkError": "Network error. Please check your connection.", + "requestTimeout": "The request timed out. Please try again.", + "unauthorized": "Your session has expired. Please sign in again.", "none": "None", "close": "Close", "back": "Back", @@ -23,7 +27,20 @@ "minutes": "minutes", "rounds": "rounds", "items": "items", - "files": "files" + "files": "files", + "notFound": "The requested resource was not found." + }, + "auth": { + "eyebrow": "Secure workspace", + "title": "Sign in to CrowdSight", + "subtitle": "Use your company account to continue.", + "email": "Email", + "emailPlaceholder": "name@company.com", + "password": "Password", + "passwordPlaceholder": "Enter your password", + "signIn": "Sign in", + "signingIn": "Signing in...", + "signInFailed": "Sign-in failed. Please try again." }, "meta": { "title": "CrowdSight - Predict Everything", @@ -404,12 +421,26 @@ "envRunning": "Environment is running and ready for Interview commands", "envNotRunningShort": "Environment not running or closed", "requireGraphIdAndQuery": "Please provide graph_id and query", + "internalError": "The server could not complete the request.", + "requestError": "The request could not be completed.", + "idempotencyRequired": "Please provide an Idempotency-Key header.", + "idempotencyInvalid": "The Idempotency-Key header is invalid.", + "idempotencyConflict": "This Idempotency-Key was already used for a different request.", + "idempotencyInProgress": "An identical request is already in progress.", + "invalidCredentials": "Invalid email or password.", + "tooManyAttempts": "Too many attempts. Please try again later.", + "organizationRequired": "Please specify organization_slug.", + "organizationNotFound": "Organization not found.", + "invalidEmail": "Please provide a valid email address.", + "invalidRole": "The requested role is not valid.", + "userExists": "A user with this email already exists.", "initReportAgent": "Initializing Report Agent..." }, "progress": { "initGraphService": "Initializing graph build service...", "textChunking": "Chunking text...", "creatingZepGraph": "Creating Zep graph...", + "creatingGraph": "Creating graph...", "settingOntology": "Setting ontology definition...", "addingChunks": "Adding {count} text chunks...", "waitingZepProcess": "Waiting for Zep to process data...", @@ -432,6 +463,7 @@ "taskFailed": "Task failed", "startPreparingEnv": "Preparing simulation environment...", "connectingZepGraph": "Connecting to Zep graph...", + "connectingGraph": "Connecting to graph...", "readingNodeData": "Reading node data...", "readingComplete": "Done, {count} entities found", "startGenerating": "Starting generation...", @@ -688,5 +720,36 @@ "fiction_story": "Fiction & Stories", "social_culture": "Social & Culture", "fillPrompt": "Edit prompt before running" + }, + "admin": { + "title": "User management", + "refresh": "Refresh", + "loading": "Loading users…", + "empty": "No users found.", + "colEmail": "Email", + "colRole": "Role", + "colStatus": "Status", + "colActions": "Actions", + "roleUser": "User", + "roleAdmin": "Admin", + "roleSuperAdmin": "Super admin", + "status_active": "Active", + "status_disabled": "Disabled", + "deactivate": "Disable", + "activate": "Activate" + }, + "settings": { + "title": "Platform LLM settings", + "provider": "Provider", + "model": "Model", + "baseUrl": "Base URL", + "apiKey": "API key", + "apiKeyPlaceholder": "•••••••• (leave empty to keep current)", + "save": "Save settings", + "saved": "Settings saved.", + "error": "Could not save settings.", + "loading": "Loading settings…", + "noneStored": "No settings stored yet.", + "version": "Version" } } \ No newline at end of file diff --git a/locales/languages.json b/locales/languages.json index 4bd404c..d3f66c4 100644 --- a/locales/languages.json +++ b/locales/languages.json @@ -1,34 +1,10 @@ { - "zh": { - "label": "中文", - "llmInstruction": "重要：你必须使用中文撰写所有内容。请使用中文回答。" - }, "en": { "label": "English", "llmInstruction": "IMPORTANT: You MUST write ALL content exclusively in English. Do NOT use any other language in your response." }, - "es": { - "label": "Español", - "llmInstruction": "Por favor, responde en español." - }, - "fr": { - "label": "Français", - "llmInstruction": "Veuillez répondre en français." - }, - "pt": { - "label": "Português", - "llmInstruction": "Por favor, responda em português." - }, - "ru": { - "label": "Русский", - "llmInstruction": "Пожалуйста, отвечайте на русском языке." - }, - "de": { - "label": "Deutsch", - "llmInstruction": "Bitte antworten Sie auf Deutsch." - }, "th": { "label": "ไทย", "llmInstruction": "IMPORTANT: You MUST write ALL content exclusively in Thai language (ภาษาไทย). Do NOT use any other language in your response. กรุณาตอบเป็นภาษาไทยเท่านั้น ห้ามใช้ภาษาอื่น" } -} \ No newline at end of file +} diff --git a/locales/th.json b/locales/th.json index 82f02ad..210d11c 100644 --- a/locales/th.json +++ b/locales/th.json @@ -13,6 +13,10 @@ "failed": "ล้มเหลว", "unknown": "ไม่ทราบ", "unknownError": "ข้อผิดพลาดที่ไม่ทราบสาเหตุ", + "requestFailed": "คำขอไม่สำเร็จ กรุณาลองใหม่อีกครั้ง", + "networkError": "เกิดข้อผิดพลาดของเครือข่าย กรุณาตรวจสอบการเชื่อมต่อ", + "requestTimeout": "คำขอใช้เวลานานเกินไป กรุณาลองใหม่อีกครั้ง", + "unauthorized": "เซสชันหมดอายุ กรุณาเข้าสู่ระบบอีกครั้ง", "none": "ไม่มี", "close": "ปิด", "back": "ย้อนกลับ", @@ -23,7 +27,20 @@ "minutes": "นาที", "rounds": "รอบ", "items": "รายการ", - "files": "ไฟล์" + "files": "ไฟล์", + "notFound": "ไม่พบทรัพยากรที่ร้องขอ" + }, + "auth": { + "eyebrow": "พื้นที่ทำงานที่ปลอดภัย", + "title": "เข้าสู่ระบบ CrowdSight", + "subtitle": "ใช้บัญชีบริษัทเพื่อดำเนินการต่อ", + "email": "อีเมล", + "emailPlaceholder": "name@company.com", + "password": "รหัสผ่าน", + "passwordPlaceholder": "กรอกรหัสผ่าน", + "signIn": "เข้าสู่ระบบ", + "signingIn": "กำลังเข้าสู่ระบบ...", + "signInFailed": "เข้าสู่ระบบไม่สำเร็จ กรุณาลองใหม่" }, "meta": { "title": "CrowdSight - พยากรณ์ทุกสิ่ง", @@ -404,12 +421,26 @@ "envRunning": "สภาพแวดล้อมกำลังทำงานและพร้อมรับคำสั่งสัมภาษณ์", "envNotRunningShort": "สภาพแวดล้อมไม่ได้ทำงานหรือถูกปิด", "requireGraphIdAndQuery": "กรุณาระบุ graph_id และ query", + "internalError": "เซิร์ฟเวอร์ไม่สามารถดำเนินการตามคำขอได้", + "requestError": "ไม่สามารถดำเนินการตามคำขอได้", + "idempotencyRequired": "กรุณาระบุเฮดเดอร์ Idempotency-Key", + "idempotencyInvalid": "ค่า Idempotency-Key ไม่ถูกต้อง", + "idempotencyConflict": "Idempotency-Key นี้ถูกใช้กับคำขออื่นแล้ว", + "idempotencyInProgress": "คำขอเดียวกันกำลังดำเนินการอยู่", + "invalidCredentials": "อีเมลหรือรหัสผ่านไม่ถูกต้อง", + "tooManyAttempts": "พยายามมากเกินไป กรุณาลองใหม่ในภายหลัง", + "organizationRequired": "กรุณาระบุ organization_slug", + "organizationNotFound": "ไม่พบองค์กร", + "invalidEmail": "กรุณาระบุอีเมลที่ถูกต้อง", + "invalidRole": "บทบาทที่ระบุไม่ถูกต้อง", + "userExists": "มีผู้ใช้ที่ใช้อีเมลนี้อยู่แล้ว", "initReportAgent": "กำลังเริ่มต้น Report Agent..." }, "progress": { "initGraphService": "กำลังเริ่มต้นบริการสร้างกราฟ...", "textChunking": "กำลังแบ่งข้อความเป็นส่วนย่อย...", "creatingZepGraph": "กำลังสร้างกราฟ Zep...", + "creatingGraph": "กำลังสร้างกราฟ...", "settingOntology": "กำลังตั้งค่าคำจำกัดความโครงสร้างข้อมูล...", "addingChunks": "กำลังเพิ่ม {count} ส่วนข้อความ...", "waitingZepProcess": "รอ Zep ประมวลผลข้อมูล...", @@ -432,6 +463,7 @@ "taskFailed": "งานล้มเหลว", "startPreparingEnv": "กำลังเตรียมสภาพแวดล้อมจำลอง...", "connectingZepGraph": "กำลังเชื่อมต่อกราฟ Zep...", + "connectingGraph": "กำลังเชื่อมต่อกราฟ...", "readingNodeData": "กำลังอ่านข้อมูลโหนด...", "readingComplete": "เสร็จ พบ {count} ตัวละคร", "startGenerating": "กำลังเริ่มสร้าง...", @@ -688,5 +720,36 @@ "fiction_story": "จำลองนิยายและเรื่องเล่า", "social_culture": "จำลองสังคมและวัฒนธรรม", "fillPrompt": "แก้ไขคำสั่งก่อนรัน" + }, + "admin": { + "title": "การจัดการผู้ใช้", + "refresh": "รีเฟรช", + "loading": "กำลังโหลดผู้ใช้…", + "empty": "ไม่พบผู้ใช้", + "colEmail": "อีเมล", + "colRole": "บทบาท", + "colStatus": "สถานะ", + "colActions": "การดำเนินการ", + "roleUser": "ผู้ใช้", + "roleAdmin": "ผู้ดูแล", + "roleSuperAdmin": "ผู้ดูแลระบบสูงสุด", + "status_active": "ใช้งาน", + "status_disabled": "ปิดใช้งาน", + "deactivate": "ปิดใช้งาน", + "activate": "เปิดใช้งาน" + }, + "settings": { + "title": "การตั้งค่าโมเดล LLM", + "provider": "ผู้ให้บริการ", + "model": "โมเดล", + "baseUrl": "Base URL", + "apiKey": "API key", + "apiKeyPlaceholder": "•••••••• (เว้นว่างเพื่อคงค่าเดิม)", + "save": "บันทึกการตั้งค่า", + "saved": "บันทึกการตั้งค่าแล้ว", + "error": "ไม่สามารถบันทึกการตั้งค่าได้", + "loading": "กำลังโหลดการตั้งค่า…", + "noneStored": "ยังไม่มีการตั้งค่า", + "version": "เวอร์ชัน" } } \ No newline at end of file diff --git a/locales/zh.json b/locales/zh.json index f15d588..0967ef4 100644 --- a/locales/zh.json +++ b/locales/zh.json @@ -1,692 +1 @@ -{ - "common": { - "confirm": "确认", - "cancel": "取消", - "loading": "加载中...", - "error": "错误", - "success": "成功", - "completed": "已完成", - "processing": "生成中", - "pending": "等待", - "ready": "就绪", - "running": "运行中", - "failed": "失败", - "unknown": "未知", - "unknownError": "未知错误", - "none": "无", - "close": "关闭", - "back": "返回", - "next": "下一步", - "retry": "重试", - "noData": "暂无数据", - "hours": "小时", - "minutes": "分钟", - "rounds": "轮", - "items": "个", - "files": "个文件" - }, - "meta": { - "title": "CrowdSight - 预测万物", - "description": "CrowdSight - 社交媒体舆论模拟系统" - }, - "nav": { - "visitGithub": "访问我们的Github主页" - }, - "home": { - "tagline": "简洁通用的群体智能引擎", - "version": "/ v0.1-预览版", - "heroTitle1": "上传任意报告", - "heroTitle2": "即刻推演未来", - "heroDesc": "即使只有一段文字，{brand} 也能基于其中的现实种子，全自动生成与之对应的至多{agentScale}构成的平行世界。通过上帝视角注入变量，在复杂的群体交互中寻找动态环境下的{optimalSolution}", - "heroDescBrand": "CrowdSight", - "heroDescAgentScale": "百万级Agent", - "heroDescOptimalSolution": "\"局部最优解\"", - "slogan": "让未来在 Agent 群中预演，让决策在百战后胜出", - "systemStatus": "系统状态", - "systemReady": "准备就绪", - "systemReadyDesc": "预测引擎待命中，可上传多份非结构化数据以初始化模拟序列", - "metricLowCost": "低成本", - "metricLowCostDesc": "常规模拟平均5$/次", - "metricHighAvail": "高可用", - "metricHighAvailDesc": "最多百万级Agent模拟", - "workflowSequence": "工作流序列", - "step01Title": "图谱构建", - "step01Desc": "现实种子提取 & 个体与群体记忆注入 & GraphRAG构建", - "step02Title": "环境搭建", - "step02Desc": "实体关系抽取 & 人设生成 & 环境配置Agent注入仿真参数", - "step03Title": "开始模拟", - "step03Desc": "双平台并行模拟 & 自动解析预测需求 & 动态更新时序记忆", - "step04Title": "报告生成", - "step04Desc": "ReportAgent拥有丰富的工具集与模拟后环境进行深度交互", - "step05Title": "深度互动", - "step05Desc": "与模拟世界中的任意一位进行对话 & 与ReportAgent进行对话", - "realitySeed": "01 / 现实种子", - "supportedFormats": "支持格式: PDF, MD, TXT", - "dragToUpload": "拖拽文件上传", - "orBrowse": "或点击浏览文件系统", - "inputParams": "输入参数", - "simulationPrompt": ">_ 02 / 模拟提示词", - "promptPlaceholder": "// 用自然语言输入模拟或预测需求（例.武大若发布撤销肖某处分的公告，会引发什么舆情走向）", - "engineBadge": "引擎: CrowdSight-V1.0", - "startEngine": "启动引擎", - "initializing": "初始化中...", - "useCases": "可以做什么", - "useCase1Title": "模拟新闻与事件", - "useCase1Desc": "上传新闻 → 模拟公众在社交媒体上的反应", - "useCase2Title": "模拟政策与法规", - "useCase2Desc": "上传政策草案 → 模拟对不同群体的影响", - "useCase3Title": "模拟商业与广告", - "useCase3Desc": "上传广告计划 → 模拟传播和受众反馈", - "useCase4Title": "模拟小说与故事", - "useCase4Desc": "上传故事 → 模拟角色行为并预测结局" - }, - "main": { - "layoutGraph": "图谱", - "layoutSplit": "双栏", - "layoutWorkbench": "工作台", - "stepNames": [ - "图谱构建", - "环境搭建", - "开始模拟", - "报告生成", - "深度互动" - ] - }, - "step1": { - "ontologyGeneration": "本体生成", - "ontologyCompleted": "已完成", - "ontologyGenerating": "生成中", - "ontologyPending": "等待", - "ontologyDesc": "LLM分析文档内容与模拟需求，提取出现实种子，自动生成合适的本体结构", - "analyzingDocs": "正在分析文档...", - "graphRagBuild": "GraphRAG构建", - "graphRagDesc": "基于生成的本体，将文档自动分块后调用 Zep 构建知识图谱，提取实体和关系，并形成时序记忆与社区摘要", - "entityNodes": "实体节点", - "relationEdges": "关系边", - "schemaTypes": "SCHEMA类型", - "buildComplete": "构建完成", - "buildCompleteDesc": "图谱构建已完成，请进入下一步进行模拟环境搭建", - "inProgress": "进行中", - "creating": "创建中...", - "enterEnvSetup": "进入环境搭建", - "createSimulationFailed": "创建模拟失败: {error}", - "createSimulationException": "创建模拟异常: {error}" - }, - "step2": { - "simInstanceInit": "模拟实例初始化", - "simInstanceDesc": "新建simulation实例，拉取模拟世界参数模版", - "asyncTaskDone": "异步任务已完成", - "generateAgentPersona": "生成 Agent 人设", - "generateAgentPersonaDesc": "结合上下文，自动调用工具从知识图谱梳理实体与关系，初始化模拟个体，并基于现实种子赋予他们独特的行为与记忆", - "currentAgentCount": "当前Agent数", - "expectedAgentTotal": "预期Agent总数", - "relatedTopicsCount": "现实种子当前关联话题数", - "generatedAgentPersonas": "已生成的 Agent 人设", - "unknownProfession": "未知职业", - "noBio": "暂无简介", - "dualPlatformConfig": "生成双平台模拟配置", - "dualPlatformConfigDesc": "LLM 根据模拟需求与现实种子，智能设置世界时间流速、推荐算法、每个个体的活跃时间段、发言频率、事件触发等参数", - "simulationDuration": "模拟时长", - "roundDuration": "每轮时长", - "totalRounds": "总轮次", - "activePerHour": "每小时活跃", - "peakHours": "高峰时段", - "workHours": "工作时段", - "morningHours": "早间时段", - "offPeakHours": "低谷时段", - "agentConfig": "Agent 配置", - "activeTimePeriod": "活跃时段", - "postsPerHour": "发帖/时", - "commentsPerHour": "评论/时", - "responseDelay": "响应延迟", - "activityLevel": "活跃度", - "sentimentBias": "情感倾向", - "influenceWeight": "影响力", - "recommendAlgoConfig": "推荐算法配置", - "platform1Name": "平台 1：广场 / 信息流", - "platform2Name": "平台 2：话题 / 社区", - "recencyWeight": "时效权重", - "popularityWeight": "热度权重", - "relevanceWeight": "相关性权重", - "viralThreshold": "病毒阈值", - "echoChamberStrength": "回音室强度", - "llmConfigReasoning": "LLM 配置推理", - "initialActivation": "初始激活编排", - "initialActivationDesc": "基于叙事方向，自动生成初始激活事件与热点话题，引导模拟世界的初始状态", - "orchestrating": "编排中", - "narrativeDirection": "叙事引导方向", - "initialHotTopics": "初始热点话题", - "initialActivationSeq": "初始激活序列 ({count})", - "setupComplete": "准备完成", - "setupCompleteDesc": "模拟环境已准备完成，可以开始运行模拟", - "roundsConfig": "模拟轮数设定", - "roundsConfigDesc": "CrowdSight 自动规划推演现实 {hours} 小时，每轮代表现实 {minutesPerRound} 分钟时间流逝", - "customToggle": "自定义", - "roundsUnit": "轮", - "estimatedDuration": "若Agent规模为100：预计耗时约 {minutes} 分钟", - "estimatedDurationFull": "若Agent规模为100：预计耗时 {minutes} 分钟", - "recommendedRounds": "{rounds} (推荐)", - "customTip": "若首次运行，强烈建议切换至'自定义模式'减少模拟轮数，以便快速预览效果并降低报错风险", - "backToGraphBuild": "返回图谱构建", - "startDualWorldSim": "开始双世界并行模拟", - "profileModalAge": "事件外显年龄", - "profileModalGender": "事件外显性别", - "profileModalCountry": "国家/地区", - "profileModalMbti": "事件外显MBTI", - "profileModalBio": "人设简介", - "profileModalTopics": "现实种子关联话题", - "profileModalPersona": "详细人设背景", - "personaDimExperience": "事件全景经历", - "personaDimExperienceDesc": "在此事件中的完整行为轨迹", - "personaDimBehavior": "行为模式侧写", - "personaDimBehaviorDesc": "经验总结与行事风格偏好", - "personaDimMemory": "独特记忆印记", - "personaDimMemoryDesc": "基于现实种子形成的记忆", - "personaDimSocial": "社会关系网络", - "personaDimSocialDesc": "个体链接与交互图谱", - "genderMale": "男", - "genderFemale": "女", - "genderOther": "其他", - "yearsOld": "岁", - "initializing": "初始化", - "generating": "生成中" - }, - "step3": { - "startGenerateReport": "开始生成结果报告", - "generatingReport": "启动中...", - "waitingForActions": "Waiting for agent actions...", - "errorMissingSimId": "错误：缺少 simulationId", - "startingDualSim": "正在启动双平台并行模拟...", - "graphMemoryUpdateEnabled": "已开启动态图谱更新模式", - "setMaxRounds": "设置最大模拟轮数: {rounds}", - "oldSimCleared": "已清理旧的模拟日志，重新开始模拟", - "engineStarted": "模拟引擎启动成功", - "startFailed": "启动失败: {error}", - "startException": "启动异常: {error}", - "stoppingSim": "正在停止模拟...", - "simStopped": "模拟已停止", - "stopFailed": "停止失败: {error}", - "stopException": "停止异常: {error}", - "allPlatformsCompleted": "检测到所有平台模拟已结束", - "simCompleted": "模拟已完成", - "graphRealtimeRefresh": "开启图谱实时刷新 (30s)", - "graphRefreshStopped": "停止图谱实时刷新", - "preparingGoBack": "准备返回 Step 2，正在关闭模拟...", - "closingSimEnv": "正在关闭模拟环境...", - "simEnvClosed": "模拟环境已关闭", - "closeFailed": "关闭模拟环境失败，尝试强制停止...", - "stoppingProcess": "正在停止模拟进程...", - "checkStatusFailed": "检查模拟状态失败: {error}", - "forceStopSuccess": "模拟已强制停止", - "forceStopFailed": "强制停止失败: {error}", - "startGenerateReportBtn": "开始生成结果报告", - "generatingReportBtn": "启动中..." - }, - "step4": { - "generatingSection": "正在生成{title}...", - "goToInteraction": "进入深度互动", - "waitingForReportAgent": "Waiting for Report Agent...", - "collapse": "收起 ▲", - "expandAll": "展开全部 {count} 条 ▼", - "expandAllEntities": "展开全部 {count} 个 ▼", - "scenarioLabel": "预测场景: ", - "tabKeyFacts": "当前关键记忆 ({count})", - "tabCoreEntities": "核心实体 ({count})", - "tabRelationChains": "关系链 ({count})", - "tabSubQueries": "子问题 ({count})", - "panelKeyFacts": "时序记忆中所关联的最新关键事实", - "totalCount": "共 {count} 条", - "totalEntityCount": "共 {count} 个", - "panelCoreEntities": "核心实体", - "factCount": "{count}条", - "panelRelationChains": "关系链", - "panelSubQueries": "漂移查询生成分析子问题", - "emptyKeyFacts": "暂无当前关键记忆", - "emptyCoreEntities": "暂无核心实体", - "emptyRelationChains": "暂无关系链", - "tabActiveFacts": "当前有效记忆 ({count})", - "tabHistoricalFacts": "历史记忆 ({count})", - "tabEntities": "涉及实体 ({count})", - "panelActiveFacts": "当前有效记忆", - "emptyActiveFacts": "暂无当前有效记忆", - "panelHistoricalFacts": "历史记忆", - "emptyHistoricalFacts": "暂无历史记忆", - "panelEntities": "涉及实体", - "emptyEntities": "暂无涉及实体", - "searchLabel": "搜索: ", - "tabFacts": "事实 ({count})", - "tabEdges": "关系 ({count})", - "tabNodes": "节点 ({count})", - "panelSearchResults": "搜索结果", - "emptySearchResults": "未找到相关结果", - "panelRelatedEdges": "相关关系", - "panelRelatedNodes": "相关节点", - "world1": "世界1", - "world2": "世界2" - }, - "step5": { - "interactiveTools": "Interactive Tools", - "agentsAvailable": "{count} agents available", - "chatWithReportAgent": "与Report Agent对话", - "chatWithAgent": "与世界中任意个体对话", - "selectChatTarget": "选择对话对象", - "sendSurvey": "发送问卷调查到世界中", - "reportAgentChat": "Report Agent - Chat", - "reportAgentDesc": "报告生成智能体的快速对话版本，可调用 4 种专业工具，拥有CrowdSight的完整记忆", - "toolInsightForge": "InsightForge 深度归因", - "toolInsightForgeDesc": "对齐现实世界种子数据与模拟环境状态，结合Global/Local Memory机制，提供跨时空的深度归因分析", - "toolPanoramaSearch": "PanoramaSearch 全景追踪", - "toolPanoramaSearchDesc": "基于图结构的广度遍历算法，重构事件传播路径，捕获全量信息流动的拓扑结构", - "toolQuickSearch": "QuickSearch 快速检索", - "toolQuickSearchDesc": "基于 GraphRAG 的即时查询接口，优化索引效率，用于快速提取具体的节点属性与离散事实", - "toolInterviewSubAgent": "InterviewSubAgent 虚拟访谈", - "toolInterviewSubAgentDesc": "自主式访谈，能够并行与模拟世界中个体进行多轮对话，采集非结构化的观点数据与心理状态", - "profileBio": "简介", - "chatEmptyReportAgent": "与 Report Agent 对话，深入了解报告内容", - "chatEmptyAgent": "与模拟个体对话，了解他们的观点", - "chatInputPlaceholder": "输入您的问题...", - "selectSurveyTarget": "选择调查对象", - "selectedCount": "已选 {selected} / {total}", - "surveyQuestions": "问卷问题", - "surveyInputPlaceholder": "输入您想问所有被选中对象的问题...", - "submitSurvey": "发送问卷", - "surveyResults": "调查结果", - "surveyResultsCount": "{count} 条回复", - "selectAll": "全选", - "clearSelection": "清空", - "errorOccurred": "抱歉，发生了错误: {error}", - "noResponse": "无响应", - "requestFailed": "请求失败", - "selectAgentFirst": "请先选择一个模拟个体", - "envNotRunning": "模拟环境未运行。问卷调查需要模拟环境处于运行状态。" - }, - "graph": { - "panelTitle": "Graph Relationship Visualization", - "refreshGraph": "刷新图谱", - "graphMemoryRealtime": "GraphRAG长短期记忆实时更新中", - "realtimeUpdating": "实时更新中...", - "pendingContentHint": "还有少量内容处理中，建议稍后手动刷新图谱", - "nodeDetails": "Node Details", - "relationship": "Relationship", - "graphDataLoading": "图谱数据加载中...", - "waitingOntology": "等待本体生成...", - "toggleMaximize": "最大化/还原", - "closeHint": "关闭提示" - }, - "history": { - "title": "推演记录", - "graphBuild": "图谱构建", - "envSetup": "环境搭建", - "analysisReport": "分析报告", - "moreFiles": "+{count} 个文件", - "noFiles": "暂无文件", - "loadingText": "加载中...", - "simRequirement": "模拟需求", - "relatedFiles": "关联文件", - "noRelatedFiles": "暂无关联文件", - "replayTitle": "推演回放", - "step1Button": "图谱构建", - "step2Button": "环境搭建", - "step4Button": "分析报告", - "replayHint": "Step3「开始模拟」与 Step5「深度互动」需在运行中启动，不支持历史回放", - "notStarted": "未开始", - "roundsProgress": "{current}/{total} 轮", - "untitledSimulation": "未命名模拟", - "unknownFile": "未知文件" - }, - "api": { - "projectNotFound": "项目不存在: {id}", - "projectDeleteFailed": "项目不存在或删除失败: {id}", - "projectDeleted": "项目已删除: {id}", - "projectReset": "项目已重置: {id}", - "requireSimulationRequirement": "请提供模拟需求描述 (simulation_requirement)", - "requireFileUpload": "请至少上传一个文档文件", - "noDocProcessed": "没有成功处理任何文档，请检查文件格式", - "requireProjectId": "请提供 project_id", - "configError": "配置错误: {details}", - "zepApiKeyMissing": "ZEP_API_KEY未配置", - "ontologyNotGenerated": "项目尚未生成本体，请先调用 /ontology/generate", - "graphBuilding": "图谱正在构建中，请勿重复提交。如需强制重建，请添加 force: true", - "textNotFound": "未找到提取的文本内容", - "ontologyNotFound": "未找到本体定义", - "graphBuildStarted": "图谱构建任务已启动，请通过 /task/{taskId} 查询进度", - "graphBuildComplete": "图谱构建完成", - "buildFailed": "构建失败: {error}", - "taskNotFound": "任务不存在: {id}", - "graphDeleted": "图谱已删除: {id}", - "entityNotFound": "实体不存在: {id}", - "graphNotBuilt": "项目尚未构建图谱，请先调用 /api/graph/build", - "requireSimulationId": "请提供 simulation_id", - "simulationNotFound": "模拟不存在: {id}", - "projectMissingRequirement": "项目缺少模拟需求描述 (simulation_requirement)", - "prepareStarted": "准备任务已启动，请通过 /api/simulation/prepare/status 查询进度", - "alreadyPrepared": "已有完成的准备工作，无需重复生成", - "notStartedPrepare": "尚未开始准备，请调用 /api/simulation/prepare 开始", - "taskCompletedPrepared": "任务已完成（准备工作已存在）", - "requireTaskOrSimId": "请提供 task_id 或 simulation_id", - "configNotFound": "模拟配置不存在，请先调用 /prepare 接口", - "configFileNotFound": "配置文件不存在，请先调用 /prepare 接口", - "unknownScript": "未知脚本: {name}，可选: {allowed}", - "scriptFileNotFound": "脚本文件不存在: {name}", - "requireGraphId": "请提供 graph_id", - "noMatchingEntities": "没有找到符合条件的实体", - "maxRoundsPositive": "max_rounds 必须是正整数", - "maxRoundsInvalid": "max_rounds 必须是有效的整数", - "invalidPlatform": "无效的平台类型: {platform}，可选: twitter/reddit/parallel", - "simRunningForceHint": "模拟正在运行中，请先调用 /stop 接口停止，或使用 force=true 强制重新开始", - "simNotReady": "模拟未准备好，当前状态: {status}，请先调用 /prepare 接口", - "graphIdRequiredForMemory": "启用图谱记忆更新需要有效的 graph_id，请确保项目已构建图谱", - "dbNotExist": "数据库不存在，模拟可能尚未运行", - "requireMessage": "请提供 message", - "missingGraphId": "缺少图谱ID", - "missingGraphIdEnsure": "缺少图谱ID，请确保已构建图谱", - "missingSimRequirement": "缺少模拟需求描述", - "reportAlreadyExists": "报告已存在", - "reportGenerateStarted": "报告生成任务已启动，请通过 /api/report/generate/status 查询进度", - "reportGenerated": "报告已生成", - "reportNotFound": "报告不存在: {id}", - "noReportForSim": "该模拟暂无报告: {id}", - "reportDeleted": "报告已删除: {id}", - "reportGenerateFailed": "报告生成失败", - "sectionNotFound": "章节不存在: section_{index}.md", - "reportProgressNotAvail": "报告不存在或进度信息不可用: {id}", - "requireAgentId": "请提供 agent_id", - "requirePrompt": "请提供 prompt（采访问题）", - "invalidInterviewPlatform": "platform 参数只能是 'twitter' 或 'reddit'", - "envNotRunning": "模拟环境未运行或已关闭。请确保模拟已完成并进入等待命令模式。", - "interviewTimeout": "等待Interview响应超时: {error}", - "requireInterviews": "请提供 interviews（采访列表）", - "interviewListMissingAgentId": "采访列表第{index}项缺少 agent_id", - "interviewListMissingPrompt": "采访列表第{index}项缺少 prompt", - "interviewListInvalidPlatform": "采访列表第{index}项的platform只能是 'twitter' 或 'reddit'", - "batchInterviewTimeout": "等待批量Interview响应超时: {error}", - "globalInterviewTimeout": "等待全局Interview响应超时: {error}", - "envRunning": "环境正在运行，可以接收Interview命令", - "envNotRunningShort": "环境未运行或已关闭", - "requireGraphIdAndQuery": "请提供 graph_id 和 query", - "initReportAgent": "初始化Report Agent..." - }, - "progress": { - "initGraphService": "初始化图谱构建服务...", - "textChunking": "文本分块中...", - "creatingZepGraph": "创建Zep图谱...", - "settingOntology": "设置本体定义...", - "addingChunks": "开始添加 {count} 个文本块...", - "waitingZepProcess": "等待Zep处理数据...", - "fetchingGraphData": "获取图谱数据...", - "graphBuildComplete": "图谱构建完成", - "buildFailed": "构建失败: {error}", - "startBuildingGraph": "开始构建图谱...", - "graphCreated": "图谱已创建: {graphId}", - "ontologySet": "本体已设置", - "textSplit": "文本已分割为 {count} 个块", - "fetchingGraphInfo": "获取图谱信息...", - "sendingBatch": "发送第 {current}/{total} 批数据 ({chunks} 块)...", - "batchFailed": "批次 {batch} 发送失败: {error}", - "noEpisodesWait": "无需等待（没有 episode）", - "waitingEpisodes": "开始等待 {count} 个文本块处理...", - "episodesTimeout": "部分文本块超时，已完成 {completed}/{total}", - "zepProcessing": "Zep处理中... {completed}/{total} 完成, {pending} 待处理 ({elapsed}秒)", - "processingComplete": "处理完成: {completed}/{total}", - "taskComplete": "任务完成", - "taskFailed": "任务失败", - "startPreparingEnv": "开始准备模拟环境...", - "connectingZepGraph": "正在连接Zep图谱...", - "readingNodeData": "正在读取节点数据...", - "readingComplete": "完成，共 {count} 个实体", - "startGenerating": "开始生成...", - "analyzingRequirements": "正在分析模拟需求...", - "generatingOutline": "正在生成报告大纲...", - "parsingOutline": "正在解析大纲结构...", - "outlinePlanComplete": "大纲规划完成", - "deepSearchAndWrite": "深度检索与撰写中 ({current}/{max})", - "initReport": "初始化报告...", - "startPlanningOutline": "开始规划报告大纲...", - "outlineDone": "大纲规划完成，共{count}个章节", - "generatingSection": "正在生成章节: {title} ({current}/{total})", - "sectionDone": "章节 {title} 已完成", - "assemblingReport": "正在组装完整报告...", - "reportComplete": "报告生成完成", - "reportFailed": "报告生成失败: {error}", - "savingProfiles": "保存Profile文件...", - "profilesComplete": "完成，共 {count} 个Profile", - "callingLLMConfig": "正在调用LLM生成配置...", - "savingConfigFiles": "正在保存配置文件...", - "configComplete": "配置生成完成", - "generatingTimeConfig": "生成时间配置...", - "generatingEventConfig": "生成事件配置和热点话题...", - "generatingAgentConfig": "生成Agent配置 ({start}-{end}/{total})...", - "generatingPlatformConfig": "生成平台配置...", - "zepSearchQuery": "关于{name}的所有信息、活动、事件、关系和背景", - "timeConfigLabel": "时间配置", - "eventConfigLabel": "事件配置", - "agentConfigResult": "Agent配置: 成功生成 {count} 个", - "postAssignResult": "初始帖子分配: {count} 个帖子已分配发布者", - "profileGenerated": "[已生成] {name} ({type})", - "readingGraphEntities": "读取图谱实体", - "generatingProfiles": "生成Agent人设", - "generatingSimConfig": "生成模拟配置", - "preparingScripts": "准备模拟脚本" - }, - "log": { - "preparingGoBack": "准备返回 Step 2，正在关闭模拟...", - "closingSimEnv": "正在关闭模拟环境...", - "simEnvClosed": "✓ 模拟环境已关闭", - "closeSimEnvFailed": "关闭模拟环境失败，尝试强制停止...", - "simForceStopSuccess": "✓ 模拟已强制停止", - "forceStopFailed": "强制停止失败: {error}", - "stoppingSimProcess": "正在停止模拟进程...", - "simStopped": "✓ 模拟已停止", - "stopSimFailed": "停止模拟失败: {error}", - "checkStatusFailed": "检查模拟状态失败: {error}", - "enterStep4": "进入 Step 4: 报告生成", - "loadingSimData": "加载模拟数据: {id}", - "timeConfig": "时间配置: 每轮 {minutes} 分钟", - "timeConfigFetchFailed": "获取时间配置失败，使用默认值: {minutes}分钟/轮", - "projectLoadSuccess": "项目加载成功: {id}", - "loadSimDataFailed": "加载模拟数据失败: {error}", - "loadException": "加载异常: {error}", - "graphDataLoadSuccess": "图谱数据加载成功", - "graphLoadFailed": "图谱加载失败: {error}", - "graphRealtimeRefreshStart": "开启图谱实时刷新 (30s)", - "graphRealtimeRefreshStop": "停止图谱实时刷新", - "simRunViewInit": "SimulationRunView 初始化", - "customRounds": "自定义模拟轮数: {rounds}", - "enterStep3": "进入 Step 3: 开始模拟", - "customRoundsConfig": "自定义模拟轮数: {rounds} 轮", - "useAutoRounds": "使用自动配置的模拟轮数", - "detectedSimEnvRunning": "检测到模拟环境正在运行，正在关闭...", - "closeSimEnvFailedWithError": "关闭模拟环境失败: {error}", - "closeSimEnvException": "关闭模拟环境异常: {error}", - "detectedSimRunning": "检测到模拟状态为运行中，正在停止...", - "forceStopSimFailed": "强制停止模拟失败: {error}", - "forceStopSimException": "强制停止模拟异常: {error}", - "simViewInit": "SimulationView 初始化", - "errorMissingSimId": "错误：缺少 simulationId", - "simInstanceCreated": "模拟实例已创建: {id}", - "preparingSimEnv": "正在准备模拟环境...", - "detectedExistingPrep": "检测到已有完成的准备工作，直接使用", - "prepareTaskStarted": "准备任务已启动", - "prepareTaskId": " └─ Task ID: {taskId}", - "zepEntitiesFound": "从Zep图谱读取到 {count} 个实体", - "entityTypes": " └─ 实体类型: {types}", - "startPollingProgress": "开始轮询准备进度...", - "prepareFailed": "准备失败: {error}", - "prepareException": "准备异常: {error}", - "prepareComplete": "✓ 准备工作已完成", - "prepareFailedWithError": "✗ 准备失败: {error}", - "startGeneratingConfig": "开始生成双平台模拟配置...", - "generatingAgentProfileConfig": "正在生成Agent人设配置...", - "generatingLLMConfig": "正在调用LLM生成模拟配置参数...", - "configComplete": "✓ 模拟配置生成完成", - "configSummaryAgents": " ├─ Agent数量: {count}个", - "configSummaryHours": " ├─ 模拟时长: {hours}小时", - "configSummaryPosts": " ├─ 初始帖子: {count}条", - "configSummaryTopics": " ├─ 热点话题: {count}个", - "configSummaryPlatforms": " └─ 平台配置: Twitter {twitter}, Reddit {reddit}", - "timeConfigDetail": "时间配置: 每轮{minutes}分钟, 共{rounds}轮", - "narrativeDirection": "叙事方向: {direction}", - "envSetupComplete": "✓ 环境搭建完成，可以开始模拟", - "startSimCustomRounds": "开始模拟，自定义轮数: {rounds} 轮", - "startSimAutoRounds": "开始模拟，使用自动配置轮数: {rounds} 轮", - "startGeneratingAgentProfiles": "开始生成Agent人设...", - "agentProfile": "→ Agent人设 {current}/{total}: {name} ({profession})", - "allProfilesComplete": "✓ 全部 {count} 个Agent人设生成完成", - "loadingExistingConfig": "正在加载已有配置数据...", - "loadedAgentProfiles": "已加载 {count} 个Agent人设", - "configLoadSuccess": "✓ 模拟配置加载成功", - "configSummaryPostsAlt": " └─ 初始帖子: {count}条", - "configGenerating": "配置生成中，开始轮询等待...", - "loadConfigFailed": "加载配置失败: {error}", - "step2Init": "Step2 环境搭建初始化", - "step3Init": "Step3 模拟运行初始化", - "startingDualSim": "正在启动双平台并行模拟...", - "setMaxRounds": "设置最大模拟轮数: {rounds}", - "graphMemoryUpdateEnabled": "已开启动态图谱更新模式", - "oldSimCleared": "✓ 已清理旧的模拟日志，重新开始模拟", - "engineStarted": "✓ 模拟引擎启动成功", - "startFailed": "✗ 启动失败: {error}", - "startException": "✗ 启动异常: {error}", - "stoppingSim": "正在停止模拟...", - "simStoppedSuccess": "✓ 模拟已停止", - "stopFailed": "停止失败: {error}", - "stopException": "停止异常: {error}", - "allPlatformsCompleted": "✓ 检测到所有平台模拟已结束", - "simCompleted": "✓ 模拟已完成", - "reportRequestSent": "报告生成请求已发送，请稍候...", - "startingReportGen": "正在启动报告生成...", - "reportGenTaskStarted": "✓ 报告生成任务已启动: {reportId}", - "reportGenFailed": "✗ 启动报告生成失败: {error}", - "reportGenException": "✗ 启动报告生成异常: {error}", - "step5Init": "Step5 深度互动初始化", - "selectChatTarget": "选择对话对象: {name}", - "sendFailed": "发送失败: {error}", - "sendToReportAgent": "向 Report Agent 发送: {message}...", - "reportAgentReplied": "Report Agent 已回复", - "sendToAgent": "向 {name} 发送: {message}...", - "agentReplied": "{name} 已回复", - "sendSurvey": "发送问卷给 {count} 个对象...", - "receivedReplies": "收到 {count} 条回复", - "surveySendFailed": "问卷发送失败: {error}", - "loadReportData": "加载报告数据: {id}", - "loadReportFailed": "加载报告失败: {error}", - "reportDataLoaded": "报告数据加载完成", - "loadReportLogFailed": "加载报告日志失败: {error}", - "loadedProfiles": "加载了 {count} 个模拟个体", - "loadProfilesFailed": "加载模拟个体失败: {error}", - "interactionViewInit": "InteractionView 初始化", - "reportViewInit": "ReportView 初始化", - "getReportInfoFailed": "获取报告信息失败: {error}", - "enterStep": "进入 Step {step}: {name}", - "returnToStep": "返回 Step {step}: {name}", - "customSimRounds": "自定义模拟轮数: {rounds} 轮" - }, - "report": { - "taskStarted": "报告生成任务开始", - "planningStart": "开始规划报告大纲", - "fetchSimContext": "获取模拟上下文信息", - "planningComplete": "大纲规划完成", - "sectionStart": "开始生成章节: {title}", - "reactThought": "ReACT 第{iteration}轮思考", - "toolCall": "调用工具: {toolName}", - "toolResult": "工具 {toolName} 返回结果", - "llmResponse": "LLM 响应 (工具调用: {hasToolCalls}, 最终答案: {hasFinalAnswer})", - "sectionContentDone": "章节 {title} 内容生成完成", - "sectionComplete": "章节 {title} 生成完成", - "reportComplete": "报告生成完成", - "errorOccurred": "发生错误: {error}", - "agentInitDone": "ReportAgent 初始化完成: graph_id={graphId}, simulation_id={simulationId}", - "executingTool": "执行工具: {toolName}, 参数: {params}", - "toolExecFailed": "工具执行失败: {toolName}, 错误: {error}", - "startPlanningOutline": "开始规划报告大纲...", - "outlinePlanDone": "大纲规划完成: {count} 个章节", - "outlinePlanFailed": "大纲规划失败: {error}", - "reactGenerateSection": "ReACT生成章节: {title}", - "sectionIterNone": "章节 {title} 第 {iteration} 次迭代: LLM 返回 None", - "sectionConflict": "章节 {title} 第 {iteration} 轮: LLM 同时输出工具调用和 Final Answer（第 {conflictCount} 次冲突）", - "sectionConflictDowngrade": "章节 {title}: 连续 {conflictCount} 次冲突，降级为截断执行第一个工具调用", - "sectionGenDone": "章节 {title} 生成完成（工具调用: {count}次）", - "multiToolOnlyFirst": "LLM 尝试调用 {total} 个工具，只执行第一个: {toolName}", - "sectionNoPrefix": "章节 {title} 未检测到 'Final Answer:' 前缀，直接采纳LLM输出作为最终内容（工具调用: {count}次）", - "sectionMaxIter": "章节 {title} 达到最大迭代次数，强制生成", - "sectionForceFailed": "章节 {title} 强制收尾时 LLM 返回 None，使用默认错误提示", - "sectionGenFailedContent": "（本章节生成失败：LLM 返回空响应，请稍后重试）", - "outlineSavedToFile": "大纲已保存到文件: {reportId}/outline.json", - "sectionSaved": "章节已保存: {reportId}/section_{sectionNum}.md", - "reportGenDone": "报告生成完成: {reportId}", - "reportGenFailed": "报告生成失败: {error}", - "agentChat": "Report Agent对话: {message}...", - "fetchReportFailed": "获取报告内容失败: {error}", - "outlineSaved": "大纲已保存: {reportId}", - "sectionFileSaved": "章节已保存: {reportId}/{fileSuffix}", - "fullReportAssembled": "完整报告已组装: {reportId}", - "reportSaved": "报告已保存: {reportId}", - "reportFolderDeleted": "报告文件夹已删除: {reportId}", - "redirectToQuickSearch": "search_graph 已重定向到 quick_search", - "redirectToInsightForge": "get_simulation_context 已重定向到 insight_forge" - }, - "console": { - "zepToolsInitialized": "ZepToolsService 初始化完成", - "zepRetryAttempt": "Zep {operation} 第 {attempt} 次尝试失败: {error}, {delay}秒后重试...", - "zepAllRetriesFailed": "Zep {operation} 在 {retries} 次尝试后仍失败: {error}", - "graphSearch": "图谱搜索: graph_id={graphId}, query={query}...", - "graphSearchOp": "图谱搜索(graph={graphId})", - "searchComplete": "搜索完成: 找到 {count} 条相关事实", - "zepSearchApiFallback": "Zep Search API失败，降级为本地搜索: {error}", - "usingLocalSearch": "使用本地搜索: query={query}...", - "localSearchComplete": "本地搜索完成: 找到 {count} 条相关事实", - "localSearchFailed": "本地搜索失败: {error}", - "fetchingAllNodes": "获取图谱 {graphId} 的所有节点...", - "fetchedNodes": "获取到 {count} 个节点", - "fetchingAllEdges": "获取图谱 {graphId} 的所有边...", - "fetchedEdges": "获取到 {count} 条边", - "fetchingNodeDetail": "获取节点详情: {uuid}...", - "fetchNodeDetailOp": "获取节点详情(uuid={uuid}...)", - "fetchNodeDetailFailed": "获取节点详情失败: {error}", - "fetchingNodeEdges": "获取节点 {uuid}... 的相关边", - "foundNodeEdges": "找到 {count} 条与节点相关的边", - "fetchNodeEdgesFailed": "获取节点边失败: {error}", - "fetchingEntitiesByType": "获取类型为 {type} 的实体...", - "foundEntitiesByType": "找到 {count} 个 {type} 类型的实体", - "fetchingEntitySummary": "获取实体 {name} 的关系摘要...", - "fetchingGraphStats": "获取图谱 {graphId} 的统计信息...", - "fetchingSimContext": "获取模拟上下文: {requirement}...", - "insightForgeStart": "InsightForge 深度洞察检索: {query}...", - "generatedSubQueries": "生成 {count} 个子问题", - "insightForgeComplete": "InsightForge完成: {facts}条事实, {entities}个实体, {relationships}条关系", - "generateSubQueriesFailed": "生成子问题失败: {error}，使用默认子问题", - "panoramaSearchStart": "PanoramaSearch 广度搜索: {query}...", - "panoramaSearchComplete": "PanoramaSearch完成: {active}条有效, {historical}条历史", - "quickSearchStart": "QuickSearch 简单搜索: {query}...", - "quickSearchComplete": "QuickSearch完成: {count}条结果", - "interviewAgentsStart": "InterviewAgents 深度采访（真实API）: {requirement}...", - "profilesNotFound": "未找到模拟 {simId} 的人设文件", - "loadedProfiles": "加载到 {count} 个Agent人设", - "selectedAgentsForInterview": "选择了 {count} 个Agent进行采访: {indices}", - "generatedInterviewQuestions": "生成了 {count} 个采访问题", - "callingBatchInterviewApi": "调用批量采访API（双平台）: {count} 个Agent", - "interviewApiReturned": "采访API返回: {count} 个结果, success={success}", - "interviewApiReturnedFailure": "采访API返回失败: {error}", - "interviewApiCallFailed": "采访API调用失败（环境未运行？）: {error}", - "interviewApiCallException": "采访API调用异常: {error}", - "interviewAgentsComplete": "InterviewAgents完成: 采访了 {count} 个Agent（双平台）", - "loadedRedditProfiles": "从 reddit_profiles.json 加载了 {count} 个人设", - "readRedditProfilesFailed": "读取 reddit_profiles.json 失败: {error}", - "loadedTwitterProfiles": "从 twitter_profiles.csv 加载了 {count} 个人设", - "readTwitterProfilesFailed": "读取 twitter_profiles.csv 失败: {error}", - "llmSelectAgentFailed": "LLM选择Agent失败，使用默认选择: {error}", - "generateInterviewQuestionsFailed": "生成采访问题失败: {error}", - "generateInterviewSummaryFailed": "生成采访摘要失败: {error}" - }, - "templates": { - "title": "选择模拟模板", - "autoSelect": "从数据自动推荐", - "custom": "自行编写", - "news_event": "新闻与事件", - "policy_regulation": "政策与法规", - "business_ad": "商业与广告", - "fiction_story": "小说与故事", - "social_culture": "社会与文化", - "fillPrompt": "运行前编辑指令" - } -} \ No newline at end of file +{}