8b84378fe1
Elevate MiroFish/CrowdSight from single-container dev to a SaaS foundation: - Local memory backend (Zep-compatible): memory services/models, local graph builder + updater, AgentActivity seam, import-boundary isolation; Zep stays default, local is opt-in behind MEMORY_BACKEND. Semantic parity not yet proven. - Durable product persistence: projects/simulations/reports schema (migration 0007) + tenant/owner-scoped ProductRepository + dual-write + scoped_project read-first + ArtifactStore abstraction; durable JobQueue + worker.py. - SaaS hardening: durable RateLimiter (wired to login), UsageService (LLM accounting), redacted AuditService, idempotency, CORS allowlist, safe API errors, single-use PasswordResetService + endpoints (covers invite-pending). - Exactly 3 roles (super_admin/admin/user) with tenant authz policy. - Admin UI: GET/POST/PATCH /api/admin/users + GET/PUT /api/admin/settings (super-admin only, encrypted/masked); AdminView.vue + SettingsView.vue with admin/super-admin route guards, th/en i18n. - Production deploy topology: multi-stage Dockerfile (frontend build + gunicorn wsgi + nginx SPA-proxy + supervisord worker), backend/wsgi.py, gunicorn dep. Backend 197 passed; frontend 10 tests + build green. ruff unavailable (gap). No commit of credentials; secrets handled via env/.env.example. Deferred: Zep semantic A/B parity, object storage cutover, mobile QA, EasyPanel container build of deploy topology.
111 lines
3.4 KiB
Python
111 lines
3.4 KiB
Python
"""Durable worker loop.
|
|
|
|
A standalone process that polls the durable ``jobs`` table, claims one job per
|
|
iteration, dispatches to a registered handler, and records completion or
|
|
failure. This is the portable core of the production worker topology and does
|
|
not depend on a specific broker; a queue provider (Redis/RabbitMQ) can be added
|
|
later by swapping the claim strategy without changing handlers.
|
|
|
|
Run with, e.g.::
|
|
PYTHONPATH=backend uv run python -m worker --once
|
|
PYTHONPATH=backend uv run python -m worker --poll-interval 1.0
|
|
"""
|
|
|
|
import argparse
|
|
import time
|
|
from typing import Callable, Optional
|
|
|
|
from app.db import create_database_engine, create_session_factory
|
|
from app.services.job_queue import JobQueue
|
|
|
|
|
|
def _make_queue():
|
|
engine = create_database_engine()
|
|
session_factory = create_session_factory(engine)
|
|
session = session_factory()
|
|
return engine, session, session_factory
|
|
|
|
|
|
def run_once(session, *, worker_id: str = "worker", organization_id: Optional[str] = None, **handlers) -> bool:
|
|
"""Claim and process one job; returns True when one was handled."""
|
|
queue = JobQueue(session)
|
|
for operation, handler in handlers.items():
|
|
queue.register_handler(operation, handler)
|
|
job = queue.claim_next_job(worker_id=worker_id, organization_id=organization_id)
|
|
if job is None:
|
|
session.rollback()
|
|
return False
|
|
try:
|
|
result = queue.dispatch(job, payload=job.result)
|
|
queue.complete_job(job.id, result=result)
|
|
session.commit()
|
|
except Exception as exc: # noqa: BLE001 - worker must not die on a bad job
|
|
session.rollback()
|
|
try:
|
|
queue.fail_job(job.id, error_code=type(exc).__name__)
|
|
session.commit()
|
|
except Exception: # noqa: BLE001
|
|
session.rollback()
|
|
return True
|
|
|
|
|
|
def run_loop(
|
|
*,
|
|
poll_interval: float = 1.0,
|
|
worker_id: str = "worker",
|
|
organization_id: Optional[str] = None,
|
|
handlers: Optional[dict[str, Callable]] = None,
|
|
) -> None:
|
|
handlers = handlers or {}
|
|
engine, session, _ = _make_queue()
|
|
try:
|
|
while True:
|
|
handled = run_once(
|
|
session,
|
|
worker_id=worker_id,
|
|
organization_id=organization_id,
|
|
**handlers,
|
|
)
|
|
if not handled:
|
|
time.sleep(poll_interval)
|
|
else:
|
|
session = _fresh_session(engine, session)
|
|
finally:
|
|
session.close()
|
|
engine.dispose()
|
|
|
|
|
|
def _fresh_session(engine, session):
|
|
try:
|
|
session.close()
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
return create_session_factory(engine)()
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Durable job worker")
|
|
parser.add_argument("--once", action="store_true", help="process a single job and exit")
|
|
parser.add_argument("--poll-interval", type=float, default=1.0)
|
|
parser.add_argument("--worker-id", default="worker")
|
|
parser.add_argument("--organization-id", default=None)
|
|
args = parser.parse_args()
|
|
|
|
if args.once:
|
|
engine, session, _ = _make_queue()
|
|
try:
|
|
run_once(session, worker_id=args.worker_id, organization_id=args.organization_id)
|
|
finally:
|
|
session.close()
|
|
engine.dispose()
|
|
else:
|
|
run_loop(
|
|
poll_interval=args.poll_interval,
|
|
worker_id=args.worker_id,
|
|
organization_id=args.organization_id,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|