8b84378fe1
Elevate MiroFish/CrowdSight from single-container dev to a SaaS foundation: - Local memory backend (Zep-compatible): memory services/models, local graph builder + updater, AgentActivity seam, import-boundary isolation; Zep stays default, local is opt-in behind MEMORY_BACKEND. Semantic parity not yet proven. - Durable product persistence: projects/simulations/reports schema (migration 0007) + tenant/owner-scoped ProductRepository + dual-write + scoped_project read-first + ArtifactStore abstraction; durable JobQueue + worker.py. - SaaS hardening: durable RateLimiter (wired to login), UsageService (LLM accounting), redacted AuditService, idempotency, CORS allowlist, safe API errors, single-use PasswordResetService + endpoints (covers invite-pending). - Exactly 3 roles (super_admin/admin/user) with tenant authz policy. - Admin UI: GET/POST/PATCH /api/admin/users + GET/PUT /api/admin/settings (super-admin only, encrypted/masked); AdminView.vue + SettingsView.vue with admin/super-admin route guards, th/en i18n. - Production deploy topology: multi-stage Dockerfile (frontend build + gunicorn wsgi + nginx SPA-proxy + supervisord worker), backend/wsgi.py, gunicorn dep. Backend 197 passed; frontend 10 tests + build green. ruff unavailable (gap). No commit of credentials; secrets handled via env/.env.example. Deferred: Zep semantic A/B parity, object storage cutover, mobile QA, EasyPanel container build of deploy topology.
121 lines
3.8 KiB
Python
121 lines
3.8 KiB
Python
"""TDD gate: durable LLM usage/cost accounting.
|
|
|
|
Records per-organization, per-user LLM usage (model, input/output tokens,
|
|
estimated cost) so admin/super-admin accounting and quotas are possible. No
|
|
prompt content or secrets are stored.
|
|
"""
|
|
|
|
import pytest
|
|
from sqlalchemy import create_engine
|
|
|
|
from app.db import Base, create_session_factory
|
|
from app.services.usage_service import UsageService
|
|
|
|
|
|
@pytest.fixture()
|
|
def session_factory():
|
|
engine = create_engine("sqlite+pysqlite:///:memory:")
|
|
Base.metadata.create_all(engine)
|
|
factory = create_session_factory(engine)
|
|
try:
|
|
yield factory
|
|
finally:
|
|
engine.dispose()
|
|
|
|
|
|
def test_usage_service_records_event(session_factory):
|
|
session = session_factory()
|
|
try:
|
|
svc = UsageService(session)
|
|
entry_id = svc.record_event(
|
|
organization_id="org-a",
|
|
user_id="user-1",
|
|
operation="report.generate",
|
|
model="gpt-4o",
|
|
input_tokens=100,
|
|
output_tokens=50,
|
|
)
|
|
assert entry_id
|
|
|
|
total = svc.total_cost(organization_id="org-a")
|
|
assert total > 0 # deterministic small estimate; no prompt text stored
|
|
|
|
rows = svc.list_events(organization_id="org-a", limit=10)
|
|
assert len(rows) == 1
|
|
assert rows[0].model == "gpt-4o"
|
|
assert rows[0].input_tokens == 100
|
|
finally:
|
|
session.close()
|
|
|
|
|
|
def test_usage_service_scopes_by_organization(session_factory):
|
|
session = session_factory()
|
|
try:
|
|
svc = UsageService(session)
|
|
svc.record_event(
|
|
organization_id="org-a", user_id="u1", operation="op", model="m",
|
|
input_tokens=1, output_tokens=1,
|
|
)
|
|
svc.record_event(
|
|
organization_id="org-b", user_id="u2", operation="op", model="m",
|
|
input_tokens=1, output_tokens=1,
|
|
)
|
|
rows_a = svc.list_events(organization_id="org-a", limit=10)
|
|
assert len(rows_a) == 1
|
|
assert rows_a[0].organization_id == "org-a"
|
|
finally:
|
|
session.close()
|
|
|
|
|
|
def test_usage_event_never_stores_prompt_or_secret(session_factory):
|
|
session = session_factory()
|
|
try:
|
|
svc = UsageService(session)
|
|
svc.record_event(
|
|
organization_id="org-a", user_id="u1", operation="report.generate",
|
|
model="m", input_tokens=1, output_tokens=1,
|
|
)
|
|
row = svc.list_events(organization_id="org-a", limit=1)[0]
|
|
blob = repr(row)
|
|
assert "prompt" not in blob.lower() # no raw prompt content persisted
|
|
finally:
|
|
session.close()
|
|
|
|
|
|
def test_usage_migration_round_trip(tmp_path, monkeypatch):
|
|
from pathlib import Path
|
|
|
|
from alembic import command
|
|
from alembic.config import Config as AlembicConfig
|
|
from sqlalchemy import inspect
|
|
|
|
from app.db import create_database_engine
|
|
|
|
database_url = f"sqlite+pysqlite:///{tmp_path / 'usage-roundtrip.db'}"
|
|
monkeypatch.setenv("DATABASE_URL", database_url)
|
|
alembic_config = AlembicConfig(str(Path(__file__).resolve().parents[1] / "alembic.ini"))
|
|
alembic_config.set_main_option("sqlalchemy.url", database_url)
|
|
|
|
command.upgrade(alembic_config, "head")
|
|
engine = create_database_engine(database_url)
|
|
try:
|
|
inspector = inspect(engine)
|
|
assert "usage_events" in set(inspector.get_table_names())
|
|
cols = {c["name"] for c in inspector.get_columns("usage_events")}
|
|
assert {
|
|
"organization_id", "user_id", "operation", "model",
|
|
"input_tokens", "output_tokens", "estimated_cost",
|
|
}.issubset(cols)
|
|
finally:
|
|
engine.dispose()
|
|
|
|
command.downgrade(alembic_config, "0009_rate_limit")
|
|
engine = create_database_engine(database_url)
|
|
try:
|
|
assert "usage_events" not in set(inspect(engine).get_table_names())
|
|
finally:
|
|
engine.dispose()
|
|
|
|
command.upgrade(alembic_config, "head")
|
|
command.check(alembic_config)
|