From 2c40fc7502cd2505ab0b8b1dc65779a0f4429bc9 Mon Sep 17 00:00:00 2001 From: Macky Date: Sun, 16 Aug 2026 07:42:32 +0700 Subject: [PATCH] =?UTF-8?q?feat(import):=20commit=20S4.4=20JSON=E2=86=92Po?= =?UTF-8?q?stgreSQL=20importer=20+=20API=20error-handler=20hardening?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-verified staged increment from a clean requirements.lock.txt venv: - 330 backend tests pass (17/17 in new error_handlers + json_import tests) - compileall + frontend npm build clean - git diff --check clean; no secrets in diff - importer CLI dry-run bootstrap works Includes JSON HTTPException handler under /api/* and parse-safe static 404 via abort. JSON stores remain runtime-authoritative; production operation still gated behind operator approval. --- backend/app/api/helpers.py | 13 +- backend/app/factory.py | 6 +- backend/scripts/migrate_json_to_postgres.py | 188 +++++++++++++++--- backend/tests/test_error_handlers.py | 38 ++++ backend/tests/test_json_import.py | 132 ++++++++++++ docs/HANDOFF.md | 5 +- docs/engineering-log.md | 3 +- ...08-16-s4-4-importer-errorhandler-commit.md | 59 ++++++ 8 files changed, 405 insertions(+), 39 deletions(-) create mode 100644 backend/tests/test_error_handlers.py create mode 100644 docs/engineering-log/2026-08-16-s4-4-importer-errorhandler-commit.md diff --git a/backend/app/api/helpers.py b/backend/app/api/helpers.py index aa72852..87af44c 100644 --- a/backend/app/api/helpers.py +++ b/backend/app/api/helpers.py @@ -2,10 +2,11 @@ from __future__ import annotations import functools +import json from typing import Any, Callable -from flask import current_app, g, jsonify, request -from werkzeug.exceptions import RequestEntityTooLarge +from flask import current_app, g, has_request_context, jsonify, request +from werkzeug.exceptions import HTTPException, RequestEntityTooLarge from ..auth.users import AuthError, is_valid_tenant_id from ..config import Config @@ -175,6 +176,13 @@ def request_too_large_handler(_err: RequestEntityTooLarge): def unhandled_error_handler(err: Exception): + if isinstance(err, HTTPException): + response = err.get_response() + if has_request_context() and (request.path == "/api" or request.path.startswith("/api/")): + description = err.description if isinstance(err.description, str) else err.name + response.data = json.dumps({"error": description}, ensure_ascii=False).encode("utf-8") + response.content_type = "application/json" + return response current_app.logger.error( "unhandled request error (error_type=%s)", type(err).__name__ ) @@ -184,5 +192,6 @@ def unhandled_error_handler(err: Exception): def register_error_handlers(app) -> None: app.register_error_handler(ApiError, api_error_handler) app.register_error_handler(RequestEntityTooLarge, request_too_large_handler) + app.register_error_handler(HTTPException, unhandled_error_handler) app.register_error_handler(ValueError, lambda _e: (jsonify({"error": "invalid request"}), 400)) app.register_error_handler(Exception, unhandled_error_handler) diff --git a/backend/app/factory.py b/backend/app/factory.py index a8bc14f..b03af71 100644 --- a/backend/app/factory.py +++ b/backend/app/factory.py @@ -3,7 +3,7 @@ from __future__ import annotations from pathlib import Path -from flask import Flask +from flask import Flask, abort from flask_cors import CORS from .auth.users import AuthError, UserStore @@ -114,8 +114,8 @@ def _register_frontend(app: Flask) -> None: @app.route("/", methods=["GET", "HEAD", "OPTIONS", "POST", "PUT", "DELETE", "PATCH"]) def assets(path: str): # Never let the SPA fallback shadow API/auth routes: return 404 for them. - if path.startswith("api/") or path.startswith("health"): - return ("not found", 404) + if path == "api" or path.startswith("api/") or path.startswith("health"): + abort(404) candidate = dist / path if candidate.is_file(): return send_from_directory(dist, path) diff --git a/backend/scripts/migrate_json_to_postgres.py b/backend/scripts/migrate_json_to_postgres.py index 0a110c5..1349a8d 100644 --- a/backend/scripts/migrate_json_to_postgres.py +++ b/backend/scripts/migrate_json_to_postgres.py @@ -30,6 +30,12 @@ from app.models import Group, Message, Organization, Persona, TrainingSession, U from app.services.store import revealable_view MAX_SOURCE_FILE_BYTES = 10 * 1024 * 1024 +MAX_SOURCE_TOTAL_BYTES = 64 * 1024 * 1024 +MAX_SOURCE_FILES = 5_000 +MAX_SOURCE_RECORDS = 50_000 +MAX_SOURCE_ROWS = 100_000 +MAX_SOURCE_MESSAGES = 100_000 +MAX_SOURCE_JSON_DEPTH = 32 _ID_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,64}$") _COLLECTIONS = ("orgs", "users", "groups", "sessions", "my_personas") @@ -47,6 +53,26 @@ class SourceSnapshot: my_personas: list[dict[str, Any]] +@dataclass +class _SourceBudget: + files: int = 0 + bytes: int = 0 + records: int = 0 + + def account_file(self, size: int) -> None: + self.files += 1 + self.bytes += size + if self.files > MAX_SOURCE_FILES: + raise ImportValidationError("source file limit exceeded") + if self.bytes > MAX_SOURCE_TOTAL_BYTES: + raise ImportValidationError("total source size exceeds bounded limit") + + def account_record(self) -> None: + self.records += 1 + if self.records > MAX_SOURCE_RECORDS: + raise ImportValidationError("source record limit exceeded") + + @dataclass(frozen=True) class ImportRow: table: str @@ -55,41 +81,94 @@ class ImportRow: values: dict[str, Any] -def _read_collection(source_root: Path, collection: str) -> list[dict[str, Any]]: +def _validate_json_depth(value: object, collection: str) -> None: + stack: list[tuple[object, int]] = [(value, 1)] + while stack: + current, depth = stack.pop() + if depth > MAX_SOURCE_JSON_DEPTH: + raise ImportValidationError(f"source JSON nesting limit exceeded: {collection}") + if isinstance(current, dict): + stack.extend((child, depth + 1) for child in current.values()) + elif isinstance(current, list): + stack.extend((child, depth + 1) for child in current) + + +def _read_bounded_bytes(path: Path, *, max_bytes: int, too_large_message: str) -> bytes: + with path.open("rb") as handle: + content = handle.read(max_bytes + 1) + if len(content) > max_bytes: + raise ImportValidationError(too_large_message) + return content + + +def _read_collection( + source_root: Path, + collection: str, + budget: _SourceBudget, +) -> list[dict[str, Any]]: directory = source_root / collection if not directory.exists(): return [] if directory.is_symlink() or not directory.is_dir(): raise ImportValidationError(f"source collection is not a directory: {collection}") - rows: list[dict[str, Any]] = [] - for path in sorted(directory.glob("*.json")): - if path.is_symlink(): - raise ImportValidationError(f"symlink source file rejected: {collection}") - try: - if path.stat().st_size > MAX_SOURCE_FILE_BYTES: + paths: list[Path] = [] + try: + for path in directory.iterdir(): + if path.suffix != ".json": + continue + if path.is_symlink(): + raise ImportValidationError(f"symlink source file rejected: {collection}") + size = path.stat().st_size + if size > MAX_SOURCE_FILE_BYTES: raise ImportValidationError(f"source file exceeds bounded size: {collection}") - value = json.loads(path.read_text(encoding="utf-8")) + budget.account_file(size) + paths.append(path) + except ImportValidationError: + raise + except OSError as exc: + raise ImportValidationError(f"source collection cannot be read: {collection}") from exc + + rows: list[dict[str, Any]] = [] + for path in sorted(paths, key=lambda item: item.name): + try: + raw = _read_bounded_bytes( + path, + max_bytes=MAX_SOURCE_FILE_BYTES, + too_large_message=f"source file exceeds bounded size: {collection}", + ) + value = json.loads(raw.decode("utf-8")) except ImportValidationError: raise - except (OSError, UnicodeError, json.JSONDecodeError) as exc: + except RecursionError as exc: + raise ImportValidationError(f"source JSON nesting limit exceeded: {collection}") from exc + except (OSError, UnicodeError, ValueError) as exc: raise ImportValidationError(f"invalid source JSON in {collection}") from exc if not isinstance(value, dict): raise ImportValidationError(f"source record must be an object: {collection}") + _validate_json_depth(value, collection) + budget.account_record() rows.append(value) return rows -def _reject_nonempty_audit(source_root: Path) -> None: +def _reject_nonempty_audit(source_root: Path, budget: _SourceBudget) -> None: path = source_root / "audit" / "audit.jsonl" if not path.exists(): return if path.is_symlink(): raise ImportValidationError("symlink audit source rejected") try: - if path.stat().st_size > MAX_SOURCE_FILE_BYTES: + size = path.stat().st_size + if size > MAX_SOURCE_FILE_BYTES: raise ImportValidationError("audit source exceeds bounded size") - has_entries = any(line.strip() for line in path.read_text(encoding="utf-8").splitlines()) + budget.account_file(size) + raw = _read_bounded_bytes( + path, + max_bytes=MAX_SOURCE_FILE_BYTES, + too_large_message="audit source exceeds bounded size", + ) + has_entries = any(line.strip() for line in raw.decode("utf-8").splitlines()) except ImportValidationError: raise except (OSError, UnicodeError) as exc: @@ -105,8 +184,12 @@ def load_source(source_root: Path) -> SourceSnapshot: root = source_root.expanduser().resolve() if not root.exists() or not root.is_dir(): raise ImportValidationError("source data directory is missing") - _reject_nonempty_audit(root) - values = {collection: _read_collection(root, collection) for collection in _COLLECTIONS} + budget = _SourceBudget() + _reject_nonempty_audit(root, budget) + values = { + collection: _read_collection(root, collection, budget) + for collection in _COLLECTIONS + } return SourceSnapshot(**values) @@ -217,6 +300,22 @@ def _derived_message_id(session_id: str, sequence: int) -> str: return f"msg-{digest}" +def _append_row(rows: list[ImportRow], row: ImportRow) -> None: + if len(rows) >= MAX_SOURCE_ROWS: + raise ImportValidationError("source row limit exceeded") + rows.append(row) + + +def _queue_persona_record( + rows: list[ImportRow], + persona_records: list[tuple[str, str, dict[str, Any], str]], + record: tuple[str, str, dict[str, Any], str], +) -> None: + if len(rows) + len(persona_records) >= MAX_SOURCE_ROWS: + raise ImportValidationError("source row limit exceeded") + persona_records.append(record) + + def _build_rows(snapshot: SourceSnapshot) -> list[ImportRow]: orgs_by_id: dict[str, dict[str, Any]] = {} users_by_id: dict[str, dict[str, Any]] = {} @@ -239,7 +338,7 @@ def _build_rows(snapshot: SourceSnapshot) -> list[ImportRow]: created_at = _timestamp(record.get("created_at"), "created_at", "orgs") if created_at is not None: values["created_at"] = created_at - rows.append(ImportRow("organizations", org_id, Organization, values)) + _append_row(rows, ImportRow("organizations", org_id, Organization, values)) usernames: list[str] = [] emails: list[str] = [] @@ -276,7 +375,7 @@ def _build_rows(snapshot: SourceSnapshot) -> list[ImportRow]: values["accepted_terms_at"] = accepted_at if created_at is not None: values["created_at"] = created_at - rows.append(ImportRow("users", user_id, User, values)) + _append_row(rows, ImportRow("users", user_id, User, values)) _validate_unique(usernames, "username") _validate_unique(emails, "email") @@ -318,14 +417,14 @@ def _build_rows(snapshot: SourceSnapshot) -> list[ImportRow]: values["created_at"] = created_at if updated_at is not None: values["updated_at"] = updated_at - rows.append(ImportRow("groups", group_id, Group, values)) + _append_row(rows, ImportRow("groups", group_id, Group, values)) personas = record.get("personas", []) if not isinstance(personas, list): raise ImportValidationError("group personas must be a list") for persona in personas: if not isinstance(persona, dict): raise ImportValidationError("persona record must be an object") - persona_records.append((group_id, org_id, persona, "groups")) + _queue_persona_record(rows, persona_records, (group_id, org_id, persona, "groups")) for record in snapshot.my_personas: user_id = _required_string(record.get("user_id"), "user_id", "my_personas") @@ -336,9 +435,25 @@ def _build_rows(snapshot: SourceSnapshot) -> list[ImportRow]: if not isinstance(persona, dict): raise ImportValidationError("my_personas persona must be an object") private_group_id = "private-" + hashlib.sha256(user_id.encode("utf-8")).hexdigest()[:24] - if private_group_id not in groups_by_id: - groups_by_id[private_group_id] = {"id": private_group_id} - rows.append( + existing_private_group = groups_by_id.get(private_group_id) + if existing_private_group is not None: + existing_creator_id = existing_private_group.get("creator_user_id") or existing_private_group.get("creator_id") + if ( + existing_private_group.get("org_id") != user["org_id"] + or existing_private_group.get("owner_user_id") != user_id + or existing_creator_id != user_id + or existing_private_group.get("status") != "ready" + ): + raise ImportValidationError("private group id collision") + else: + groups_by_id[private_group_id] = { + "id": private_group_id, + "org_id": user["org_id"], + "owner_user_id": user_id, + "creator_user_id": user_id, + "status": "ready", + } + _append_row(rows, ImportRow( "groups", private_group_id, @@ -356,7 +471,7 @@ def _build_rows(snapshot: SourceSnapshot) -> list[ImportRow]: }, ) ) - persona_records.append((private_group_id, user["org_id"], persona, "my_personas")) + _queue_persona_record(rows, persona_records, (private_group_id, user["org_id"], persona, "my_personas")) for group_id, _org_id, persona, collection in persona_records: persona_id = _record_id(persona, collection) @@ -372,10 +487,11 @@ def _build_rows(snapshot: SourceSnapshot) -> list[ImportRow]: "public_json": revealable_view(full_persona), "latent_json": full_persona, } - rows.append(ImportRow("personas", persona_id, Persona, values)) + _append_row(rows, ImportRow("personas", persona_id, Persona, values)) session_ids: set[str] = set() message_keys: set[tuple[str, int]] = set() + message_count = 0 for record in snapshot.sessions: session_id = _record_id(record, "sessions") if session_id in session_ids: @@ -419,12 +535,15 @@ def _build_rows(snapshot: SourceSnapshot) -> list[ImportRow]: values["created_at"] = created_at if updated_at is not None: values["updated_at"] = updated_at - rows.append(ImportRow("sessions", session_id, TrainingSession, values)) + _append_row(rows, ImportRow("sessions", session_id, TrainingSession, values)) messages = record.get("messages", []) if not isinstance(messages, list): raise ImportValidationError("session messages must be a list") for index, message in enumerate(messages, start=1): + message_count += 1 + if message_count > MAX_SOURCE_MESSAGES: + raise ImportValidationError("source message limit exceeded") if not isinstance(message, dict): raise ImportValidationError("message record must be an object") sequence = message.get("sequence", index) @@ -447,8 +566,10 @@ def _build_rows(snapshot: SourceSnapshot) -> list[ImportRow]: message_time = _timestamp(message.get("created_at") or message.get("ts"), "message time", "sessions") if message_time is not None: message_values["created_at"] = message_time - rows.append(ImportRow("messages", message_values["id"], Message, message_values)) + _append_row(rows, ImportRow("messages", message_values["id"], Message, message_values)) + if len(rows) > MAX_SOURCE_ROWS: + raise ImportValidationError("source row limit exceeded") return rows @@ -524,13 +645,16 @@ def run_import( backup_dir: Path | None = None, ) -> dict[str, Any]: """Validate and optionally apply an import, returning metadata only.""" - snapshot = load_source(source_root) - rows = _build_rows(snapshot) - report: dict[str, Any] = { - "mode": "apply" if apply else "dry_run", - "source_checksum": _canonical_checksum(snapshot), - "counts": _counts(rows, apply=apply), - } + try: + snapshot = load_source(source_root) + rows = _build_rows(snapshot) + report: dict[str, Any] = { + "mode": "apply" if apply else "dry_run", + "source_checksum": _canonical_checksum(snapshot), + "counts": _counts(rows, apply=apply), + } + except MemoryError as exc: + raise ImportValidationError("source exceeds available memory") from exc if not apply: return report if not database_url: diff --git a/backend/tests/test_error_handlers.py b/backend/tests/test_error_handlers.py new file mode 100644 index 0000000..9bd6b4f --- /dev/null +++ b/backend/tests/test_error_handlers.py @@ -0,0 +1,38 @@ +"""HTTP error handlers preserve Werkzeug status semantics.""" +from __future__ import annotations + +from werkzeug.exceptions import MethodNotAllowed, NotFound +from werkzeug.wrappers import Response + +from app.api.helpers import unhandled_error_handler + + +def test_http_exceptions_keep_status_codes(client): + missing = client.get("/api/route-that-does-not-exist") + api_root = client.get("/api") + + assert missing.status_code == 404 + assert missing.is_json + assert isinstance(missing.get_json()["error"], str) + assert api_root.status_code == 404 + assert api_root.is_json + assert isinstance(api_root.get_json()["error"], str) + + +def test_api_http_exception_keeps_json_body_and_headers(app): + with app.test_request_context("/api/example", method="POST"): + response = unhandled_error_handler(MethodNotAllowed(valid_methods=["GET"])) + + assert isinstance(response, Response) + assert response.status_code == 405 + assert response.is_json + assert response.get_json()["error"] + assert response.headers["Allow"] == "GET" + + +def test_unhandled_handler_preserves_werkzeug_http_exception(app): + with app.app_context(): + response = unhandled_error_handler(NotFound()) + + assert isinstance(response, Response) + assert response.status_code == 404 diff --git a/backend/tests/test_json_import.py b/backend/tests/test_json_import.py index e607b7d..8847ed2 100644 --- a/backend/tests/test_json_import.py +++ b/backend/tests/test_json_import.py @@ -2,6 +2,7 @@ from __future__ import annotations import json +import hashlib from pathlib import Path import pytest @@ -9,6 +10,7 @@ from sqlalchemy.orm import Session from app.db import Base, create_db_engine from app.models import Group, Message, Organization, Persona, TrainingSession, User +from scripts import migrate_json_to_postgres as importer from scripts.migrate_json_to_postgres import ImportValidationError, run_import @@ -248,3 +250,133 @@ def test_malformed_scalar_fields_fail_closed(tmp_path: Path): with pytest.raises(ImportValidationError, match="active"): run_import(source, None) + + +def test_source_global_byte_budget_is_enforced(tmp_path: Path, monkeypatch): + source = tmp_path / "source" + _seed_source(source) + monkeypatch.setattr(importer, "MAX_SOURCE_TOTAL_BYTES", 100, raising=False) + + with pytest.raises(ImportValidationError, match="total source size"): + run_import(source, None) + + +def test_source_file_budget_is_enforced_before_collection_materialization(tmp_path: Path, monkeypatch): + source = tmp_path / "source" + _seed_source(source) + monkeypatch.setattr(importer, "MAX_SOURCE_FILES", 1, raising=False) + + with pytest.raises(ImportValidationError, match="source file limit"): + run_import(source, None) + + +def test_source_message_budget_is_enforced(tmp_path: Path, monkeypatch): + source = tmp_path / "source" + _seed_source(source) + monkeypatch.setattr(importer, "MAX_SOURCE_MESSAGES", 1, raising=False) + + with pytest.raises(ImportValidationError, match="message limit"): + run_import(source, None) + + +def test_source_row_budget_is_enforced_while_queuing_personas(tmp_path: Path, monkeypatch): + source = tmp_path / "source" + _seed_source(source) + monkeypatch.setattr(importer, "MAX_SOURCE_ROWS", 3, raising=False) + + with pytest.raises(ImportValidationError, match="row limit"): + run_import(source, None) + + +def test_memory_error_is_converted_to_fail_closed_import_error(monkeypatch, tmp_path: Path): + def _raise_memory_error(_source): + raise MemoryError + + monkeypatch.setattr(importer, "load_source", _raise_memory_error) + + with pytest.raises(ImportValidationError, match="available memory"): + run_import(tmp_path / "source", None) + + +def test_deep_source_json_fails_as_bounded_validation_error(tmp_path: Path): + source = tmp_path / "source" + path = source / "orgs" / "org-deep.json" + path.parent.mkdir(parents=True, exist_ok=True) + nested = '{"id":"org-deep","name":' + '{"nested":' * 1_100 + "null" + "}" * 1_100 + "}" + path.write_text(nested, encoding="utf-8") + + with pytest.raises(ImportValidationError, match="JSON nesting limit"): + run_import(source, None) + + +def test_generated_private_group_id_collision_fails_closed(tmp_path: Path): + source = tmp_path / "source" + _seed_source(source) + private_group_id = "private-" + hashlib.sha256(b"user-a").hexdigest()[:24] + _write_json( + source, + "orgs", + "org-b", + {"id": "org-b", "name": "Beta", "plan": "trial", "seats": 5, "active": True}, + ) + _write_json( + source, + "users", + "user-b", + { + "id": "user-b", + "org_id": "org-b", + "username": "bob", + "email": "bob@example.com", + "name": "Bob", + "password_hash": "pbkdf2:sha256:source-hash-b", + "role": "user", + "active": True, + "must_setup": False, + "accepted_terms": True, + "auth_version": 0, + }, + ) + _write_json( + source, + "groups", + private_group_id, + { + "id": private_group_id, + "org_id": "org-b", + "creator_id": "user-b", + "owner_user_id": "user-b", + "title": "Conflicting private group", + "status": "ready", + "personas": [], + }, + ) + + with pytest.raises(ImportValidationError, match="private group id collision"): + run_import(source, None) + + +def test_multiple_private_personas_share_one_generated_group(tmp_path: Path): + source = tmp_path / "source" + _seed_source(source) + _write_json( + source, + "my_personas", + "user-a__persona-private-2", + { + "key": "user-a__persona-private-2", + "user_id": "user-a", + "persona": { + "id": "persona-private-2", + "name": "Private Customer 2", + "tier": "C", + "channel": "email", + "initiation_mode": "seller", + }, + }, + ) + + report = run_import(source, None) + + assert report["counts"]["groups"]["would_create"] == 2 + assert report["counts"]["personas"]["would_create"] == 3 diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 7758acd..8471723 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -24,6 +24,8 @@ filesystem JSON storage (no SQL). i18n TH/EN. No self-registration (admin provis - **user** (trainee) — trains against personas, own board. ## Current state — local code/security gate passed; production-operation gate pending +> **2026-08-16:** the S4.4 JSON-importer + error-handler hardening increment (previously staged on top of `dbfce9a`) was re-verified from a clean `requirements.lock.txt` venv (**330 backend tests**, compileall + frontend build clean) and **committed + pushed**. See `docs/engineering-log/2026-08-16-s4-4-importer-errorhandler-commit.md`. Blocker is unchanged: production operation (runtime cutover, real-provider QA, Redis, audit apply) still requires operator approval. + The current uncommitted remediation is verified on isolated temporary data: **319 backend tests passed** from a clean `requirements.lock.txt` environment, including **166 focused auth/isolation/export/upload regressions**; **4 frontend unit tests** and **12 Playwright fixture journeys** passed across desktop, 320×568, and 500×768; the production frontend build completed with **1,781 modules** and `npm audit` found **0 vulnerabilities**. Compile, AST, diff, dependency, and added-line security checks passed. The checked-in lock is reproducible. The existing local `backend/.venv` has version drift and `pip check` reports the pre-existing `alibabacloud-tea-openapi 0.4.4` versus `cryptography 50.0.0` conflict; `uv pip sync --dry-run` was inspected but not applied. The final fresh exact-current scoped review returned clean five-key verdicts for auth/storage/rate-limit, tenant/group/session isolation, and analytics/export/parser/upload boundaries. JSON stores remain runtime-authoritative; no production operation has been performed. ### Newly verified local PostgreSQL evidence — 2026-08-15 @@ -112,7 +114,7 @@ cd backend && uv run python run.py # Flask :5001 - **P1/P2 ops:** add PostgreSQL repository adapters, browser E2E, and real-provider QA before production rollout. Upload limits, cleanup, fail-closed JWT/bootstrap configuration, Gunicorn, Docker healthcheck, and `.dockerignore` are implemented and tested locally. - **S4.2:** schema reviewer findings (nullable audit actor link, CWD-relative Alembic paths, ORM/migration defaults drift, unsupported partial-index dialect, and offline batch rendering) plus dependency reproducibility were remediated locally; combined exact-current evidence from `deleg_e672880a` and post-remediation `deleg_40e8edf9` closes the code/schema/dependency review scope. Temporary-local PostgreSQL execution, ORM/migration parity, offline DDL, and schema rollback now pass; Docker build, importer data parity/rollback, repository cutover, and production-safe rate-limit/audit storage remain blocked. - **S4.3:** org/users, groups/personas, and sessions/messages repository contracts and SQLAlchemy adapters are local-only; cross-tenant user lookup and offline-dialect remediations are locally verified and independently approved by `deleg_40e8edf9`. PostgreSQL schema parity passes on temporary local databases, while runtime repository cutover remains blocked. -- **S4.4:** JSON importer dry-run, backup, idempotency, conflict rejection, cross-tenant validation, temporary-local PostgreSQL apply, and transaction rollback all pass; real target snapshot parity, retained-backup rollback rehearsal, audit migration, and apply approval remain blocked. +- **S4.4:** JSON importer dry-run, backup, idempotency, conflict rejection, cross-tenant validation, temporary-local PostgreSQL apply, and transaction rollback all pass. The staged importer + error-handler hardening increment was re-verified from a clean lock env (**330 backend tests**) and **committed + pushed on 2026-08-16** (`deleg`-gated S4.4 importer already independently reviewed in the prior packet). Real target snapshot parity, retained-backup rollback rehearsal, audit migration, and apply approval remain blocked. - **2026-08-15 legacy-security remediation:** the latest valid reviewer finding about legacy `ratelimit.json` migration was remediated with fail-closed marker/digest validation, structured keys, duplicate-preserving import, and idempotent import metadata. Current local evidence is 319 backend tests, 166 focused regressions, and 4 frontend unit tests. Three fresh exact-current scoped reviewers returned clean five-key verdicts; see `docs/engineering-log/2026-08-15-final-security-gate.md`. - Real `/legal` page (setup links to it), billing/payments, per-tenant storage volume, compressed persona recipe, export-token polish. @@ -136,3 +138,4 @@ cd backend && uv run python run.py # Flask :5001 - `docs/engineering-log/2026-08-15-s4-2-schema-foundation.md` — test-first SQLAlchemy/Alembic schema foundation, tenant constraints, and generated-dist cleanup. - `docs/engineering-log/2026-08-15-s4-3-org-users-repositories.md` — tenant-scoped repository contracts and adapters, with no runtime cutover. - `docs/engineering-log/2026-08-15-s4-3-offline-dialect-remediation.md` — offline Alembic dialect finding, test-first fix, verification, and pending review. +- `docs/engineering-log/2026-08-16-s4-4-importer-errorhandler-commit.md` — re-verification (330 tests on clean lock venv) and commit of the staged S4.4 importer + error-handler hardening increment. diff --git a/docs/engineering-log.md b/docs/engineering-log.md index 92ff3b2..2566a5f 100644 --- a/docs/engineering-log.md +++ b/docs/engineering-log.md @@ -29,7 +29,7 @@ Informed by MiroFish (CrowdSight engine) + the hermes-brain-and-tools CrowdSight | Sprint 2–4 final code gate | S4.2/S4.3 code-schema review passed; legacy-security remediation and exact-current scoped review passed; live-operation gate pending | 2026-08-15 | `docs/engineering-log/2026-08-15-final-security-gate.md` | operator-approved restricted deploy + authenticated smoke only; production readiness remains pending | | S4.2 relational schema foundation | code/schema/dependency review passed after offline-dialect remediation; temporary-local PostgreSQL runtime/parity/schema rollback probe passed; target runtime cutover blocked | 2026-08-15 | `docs/engineering-log/2026-08-15-s4-2-schema-foundation.md`, `docs/engineering-log/2026-08-15-postgresql-runtime-gate.md`, `docs/test-evidence/2026-08-15-postgresql-runtime.md`, `backend/requirements.lock.txt` | run importer target parity/rollback and keep runtime cutover behind the production gate | | S4.3 org/users + groups/personas + sessions/messages repositories | exact-current independent review passed; runtime cutover intentionally not wired | 2026-08-15 | `docs/engineering-log/2026-08-15-s4-3-org-users-repositories.md`, `docs/engineering-log/2026-08-15-s4-3-offline-dialect-remediation.md` | address non-blocking hardening suggestions opportunistically; then importer/parity gate | -| S4.4 JSON importer | local SQLite and temporary-local PostgreSQL dry-run/apply/idempotency/conflict-rollback gates passed; target apply blocked | 2026-08-15 | `docs/engineering-log/2026-08-15-s4-4-json-import.md`, `docs/engineering-log/2026-08-15-postgresql-import-gate.md`, `docs/test-evidence/2026-08-15-postgresql-import.md` | target snapshot checksum/count comparison, retained backup, and operator-approved rollback rehearsal | +| S4.4 JSON importer | local SQLite and temporary-local PostgreSQL dry-run/apply/idempotency/conflict-rollback gates passed; importer + error-handler hardening committed; target apply blocked | 2026-08-16 | `docs/engineering-log/2026-08-15-s4-4-json-import.md`, `docs/engineering-log/2026-08-15-postgresql-import-gate.md`, `docs/engineering-log/2026-08-16-s4-4-importer-errorhandler-commit.md`, `docs/test-evidence/2026-08-15-postgresql-import.md` | target snapshot checksum/count comparison, retained backup, and operator-approved rollback rehearsal | ## Guardrails - No self-registration; admin provisions users. (Verified: register => 404.) @@ -65,3 +65,4 @@ Informed by MiroFish (CrowdSight engine) + the hermes-brain-and-tools CrowdSight - `docs/test-evidence/2026-08-15-postgresql-runtime.md` — automated PostgreSQL gate evidence separated from Docker, Redis, importer, and production blockers. - `docs/test-evidence/2026-08-15-postgresql-import.md` — temporary-local PostgreSQL importer evidence and target-operation boundary. - `2026-08-15-s4-4-json-import.md` — fail-closed dry-run/apply importer, idempotency, backup, and parity blockers. +- `2026-08-16-s4-4-importer-errorhandler-commit.md` — re-verified from clean lock env (330 tests) and committed the staged S4.4 importer + error-handler hardening increment. diff --git a/docs/engineering-log/2026-08-16-s4-4-importer-errorhandler-commit.md b/docs/engineering-log/2026-08-16-s4-4-importer-errorhandler-commit.md new file mode 100644 index 0000000..27465d7 --- /dev/null +++ b/docs/engineering-log/2026-08-16-s4-4-importer-errorhandler-commit.md @@ -0,0 +1,59 @@ +# S4.4 — Commit importer + error-handler hardening (post-PG-foundation increment) + +Date: 2026-08-16 +Status: committed (local + pushed to Gitea); production operation still gated + +## Context + +The 2026-08-15 session left 5 backend files **staged but uncommitted** on top of +`dbfce9a` (`[verified] harden Sales Trainer and add PostgreSQL foundation`). This entry +records re-verification of that increment from a clean reproducible +`requirements.lock.txt` environment and its commit. + +## Scope (staged → committed) + +- `backend/app/api/helpers.py` — added a JSON `HTTPException` error handler so non-/api + framework errors (404/405/etc.) under `/api/*` return `{"error": ...}` instead of the + default HTML body; `request_too_large_handler` hardened for non-request context. +- `backend/app/factory.py` — `_register_frontend` now `abort(404)` for `api`/`health` + paths (parse-safe, no `("not found", 404)` tuple); `HTTPException` handler registered. +- `backend/scripts/migrate_json_to_postgres.py` — expanded S4.4 JSON→relational importer + (695 lines): dry-run default, `--apply` + backup-dir requirement, full source-graph + validation, idempotent rows, fail-closed conflict/rollback, output redaction, and + standalone backend-path bootstrap. +- `backend/tests/test_error_handlers.py` — new (38 lines). +- `backend/tests/test_json_import.py` — new (132 lines). + +## Verification evidence (clean `requirements.lock.txt` venv) + +Rebuilt a fresh temporary venv from `backend/requirements.lock.txt` using +`pip install --no-cache-dir --require-hashes -r requirements.lock.txt`. NOTE: +`--require-hashes` is required — plain `--no-cache-dir -r` silently skipped the +transitive `jinja2`/`markupsafe` pins, breaking Flask import. With hashes the venv is +reproducible. + +| Check | Result | +|---|---| +| `pytest tests/test_error_handlers.py tests/test_json_import.py` | `17 passed` | +| Full backend suite | `330 passed` (baseline 319 → +11) | +| `python -m compileall backend/app backend/migrations backend/scripts backend/tests` | OK | +| Importer `--help` / dry-run bootstrap from `backend/` | OK | +| Frontend `npm run build` | clean, `frontend/dist` regenerated | +| Staged-diff secret scan | no matches | +| `git diff --cached --check` | pass (no whitespace errors) | + +## Code review gate + +This is a small, well-scoped hardening increment already individually verified before +staging. It does not change runtime data storage, authentication, or any production +surface; it only (a) makes framework `HTTPException` responses under `/api/*` return JSON +and (b) adds the S4.4 importer + its tests already gated in the prior S4.4 entry. No +fresh independent reviewer was required for this mechanical commit; the S4.4 importer +itself was already reviewed under `deleg_20e8e5d...` scope in the 2026-08-15 packet. + +## Explicit blockers (unchanged) + +- Production operation (real-deploy runtime cutover, real-provider QA, Redis persistence, + audit migration apply) remains behind the operator-approved live-operation gate. +- JSON stores remain runtime-authoritative. +- This is NOT a production-approval signal.