"""Frozen, replayable Tourism research runs. A run either produces a deterministic event-study result from point-in-time inputs or persists a blocked report explaining the exact missing gate. It never falls back to revised vendor history. """ from __future__ import annotations import copy import hashlib import json import math import re from datetime import datetime, timezone from pathlib import Path from typing import Any, Mapping, Sequence from .event_study import EventStudyError, _canonical_timestamp, assess_backtest_readiness, run_event_study from .prices import PriceSnapshotStore, PriceSourceError from .tourism import compute_tourism_signal from .vintages import VintageStore, VintageStoreError RUN_SCHEMA_VERSION = 1 _RUN_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}") class ResearchRunError(ValueError): """Raised when a research run cannot be safely created or replayed.""" def _canonical_hash(payload: Mapping[str, Any]) -> str: encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") return hashlib.sha256(encoded).hexdigest() def _atomic_write(path: Path, payload: Mapping[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.tmp") temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") temporary.replace(path) class ResearchRunStore: """Immutable JSON reports with a compact run manifest.""" def __init__(self, root: Path | str) -> None: self.root = Path(root).resolve() self.report_dir = self.root / "reports" self.manifest_path = self.root / "manifest.json" def load_manifest(self) -> dict[str, Any]: if not self.manifest_path.is_file(): return {"schema_version": RUN_SCHEMA_VERSION, "runs": {}} try: manifest = json.loads(self.manifest_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise ResearchRunError("research run manifest is unreadable") from exc if manifest.get("schema_version") != RUN_SCHEMA_VERSION or not isinstance(manifest.get("runs"), dict): raise ResearchRunError("unsupported research run manifest schema") return manifest def _report_path(self, run_id: str) -> Path: if not _RUN_ID_RE.fullmatch(run_id): raise ResearchRunError("invalid research run id") candidate = (self.report_dir / f"{run_id}.json").resolve() if candidate.parent != self.report_dir.resolve(): raise ResearchRunError("research report path escaped store root") return candidate def load(self, run_id: str) -> dict[str, Any]: path = self._report_path(run_id) if not path.is_file(): raise FileNotFoundError(run_id) try: report = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise ResearchRunError("research report is unreadable") from exc if report.get("run_id") != run_id or report.get("schema_version") != RUN_SCHEMA_VERSION: raise ResearchRunError("research report identity mismatch") return report def persist(self, report: dict[str, Any]) -> dict[str, Any]: run_id = str(report.get("run_id", "")) path = self._report_path(run_id) manifest = self.load_manifest() existing = manifest["runs"].get(run_id) if existing: stored = self.load(run_id) if stored != report: raise ResearchRunError("research run id already contains a different report") return stored stored = copy.deepcopy(report) _atomic_write(path, stored) manifest["runs"][run_id] = { "run_id": run_id, "status": stored.get("status"), "reason": stored.get("reason"), "generated_at": stored.get("generated_at"), "report_file": path.name, } _atomic_write(self.manifest_path, manifest) return stored def latest(self) -> dict[str, Any]: entries = list(self.load_manifest().get("runs", {}).values()) if not entries: raise FileNotFoundError("no research runs") latest = max(entries, key=lambda item: (str(item.get("generated_at", "")), str(item.get("run_id", "")))) return self.load(str(latest["run_id"])) def list_runs(self) -> list[dict[str, Any]]: entries = list(self.load_manifest().get("runs", {}).values()) return sorted(entries, key=lambda item: (str(item.get("generated_at", "")), str(item.get("run_id", ""))), reverse=True) def _validate_config(windows: Sequence[int], cost_bps: float, min_events: int, execution_lag_sessions: int) -> tuple[list[int], float, int, int]: if not windows or any(isinstance(window, bool) or int(window) != window or int(window) <= 0 for window in windows): raise ResearchRunError("windows must contain positive integers") try: parsed_cost = float(cost_bps) except (TypeError, ValueError) as exc: raise ResearchRunError("cost_bps must be finite and non-negative") from exc if not math.isfinite(parsed_cost) or parsed_cost < 0: raise ResearchRunError("cost_bps must be finite and non-negative") if isinstance(min_events, bool) or int(min_events) != min_events or int(min_events) < 1: raise ResearchRunError("min_events must be positive") if isinstance(execution_lag_sessions, bool) or int(execution_lag_sessions) != execution_lag_sessions or int(execution_lag_sessions) < 0: raise ResearchRunError("execution_lag_sessions must be a non-negative integer") return [int(window) for window in windows], parsed_cost, int(min_events), int(execution_lag_sessions) def _price_gate(price_entry: Mapping[str, Any] | None) -> dict[str, Any]: if not price_entry: return {"status": "blocked", "reason": "price_snapshot_missing", "snapshot_count": 0} if price_entry.get("point_in_time") is not True: return { "status": "blocked", "reason": "price_series_not_point_in_time", "snapshot_id": price_entry.get("snapshot_id"), "quality": price_entry.get("quality"), "point_in_time": False, } return { "status": "ready", "reason": None, "snapshot_id": price_entry.get("snapshot_id"), "quality": price_entry.get("quality"), "point_in_time": True, } def _revision_key(entry: Mapping[str, Any]) -> tuple[str, str, str]: source_id = str(entry.get("source_id") or "") published_at = _canonical_timestamp(str(entry.get("published_at") or "")) if source_id and published_at: return ("release", source_id, published_at) return ("id", str(entry.get("vintage_id") or ""), "") def _select_independent_vintages(entries: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: selected: dict[tuple[str, str, str], dict[str, Any]] = {} for raw_entry in entries: entry = dict(raw_entry) key = _revision_key(entry) current = selected.get(key) candidate_rank = (str(entry.get("first_seen_at", "")), str(entry.get("last_seen_at", "")), str(entry.get("vintage_id", ""))) current_rank = (str(current.get("first_seen_at", "")), str(current.get("last_seen_at", "")), str(current.get("vintage_id", ""))) if current else ("", "", "") if current is None or candidate_rank > current_rank: selected[key] = entry return sorted(selected.values(), key=lambda item: (str(item.get("published_at", "")), str(item.get("vintage_id", "")))) def _series_rows(series: Mapping[str, Any], price_field: str, symbol: str) -> list[dict[str, Any]]: bars = series.get("bars") if not isinstance(bars, list) or not bars: raise ResearchRunError(f"missing price bars for {symbol}") rows: list[dict[str, Any]] = [] for bar in bars: if not isinstance(bar, dict) or price_field not in bar: raise ResearchRunError(f"price field {price_field} missing for {symbol}") rows.append({"date": bar.get("date"), "close": bar.get(price_field)}) return rows def run_tourism_research( vintage_store: VintageStore, price_store: PriceSnapshotStore, run_store: ResearchRunStore, *, min_events: int = 12, windows: Sequence[int] = (1, 3, 5, 20), cost_bps: float = 20.0, execution_lag_sessions: int = 1, ) -> dict[str, Any]: windows, cost_bps, min_events, execution_lag_sessions = _validate_config(windows, cost_bps, min_events, execution_lag_sessions) try: vintages = _select_independent_vintages(vintage_store.list_vintages()) price_entry = price_store.latest_snapshot_entry() except (VintageStoreError, PriceSourceError) as exc: raise ResearchRunError(str(exc)) from exc vintage_gate = assess_backtest_readiness(vintages, min_events=min_events) validated_vintage_snapshots: dict[str, dict[str, Any]] = {} try: for entry in vintages: vintage_id = str(entry["vintage_id"]) validated_vintage_snapshots[vintage_id] = vintage_store.load_snapshot(vintage_id) except (FileNotFoundError, VintageStoreError, KeyError) as exc: raise ResearchRunError(f"vintage snapshot integrity validation failed: {exc}") from exc price_snapshot: dict[str, Any] | None = None price_gate_entry = price_entry if price_entry: try: price_snapshot = price_store.load_snapshot(str(price_entry["snapshot_id"])) except (FileNotFoundError, PriceSourceError, KeyError) as exc: raise ResearchRunError(f"price snapshot integrity validation failed: {exc}") from exc price_source = price_snapshot.get("source", {}) price_gate_entry = { **price_entry, "point_in_time": price_source.get("point_in_time"), "quality": price_source.get("quality"), "normalized_snapshot_hash": price_source.get("normalized_snapshot_hash"), } price_gate = _price_gate(price_gate_entry) input_vintages = [ { "vintage_id": entry.get("vintage_id"), "source_id": entry.get("source_id"), "as_of": entry.get("as_of"), "published_at": entry.get("published_at"), "raw_payload_hash": entry.get("raw_payload_hash"), "parser_version": entry.get("parser_version"), "revision_status": entry.get("revision_status"), } for entry in vintages ] input_price = copy.deepcopy(price_gate_entry) if price_gate_entry else None config = { "min_events": min_events, "windows": windows, "cost_bps": cost_bps, "execution_lag_sessions": execution_lag_sessions, } input_fingerprint = _canonical_hash({"theme": "tourism", "config": config, "vintages": input_vintages, "price_snapshot": input_price}) run_id = f"tourism-run-{input_fingerprint[:16]}" try: return run_store.load(run_id) except FileNotFoundError: pass generated_at = datetime.now(timezone.utc).isoformat() report: dict[str, Any] = { "schema_version": RUN_SCHEMA_VERSION, "run_id": run_id, "theme": "tourism", "status": "blocked", "reason": None, "generated_at": generated_at, "config": config, "gates": {"vintages": vintage_gate, "prices": price_gate}, "inputs": {"vintages": input_vintages, "price_snapshot": input_price}, } if vintage_gate["status"] != "ready": report["reason"] = vintage_gate["reason"] return run_store.persist(report) if price_gate["status"] != "ready": report["reason"] = price_gate["reason"] return run_store.persist(report) if not price_entry or price_snapshot is None: raise ResearchRunError("price snapshot disappeared after readiness check") price_source = price_snapshot.get("source", {}) price_field = str(price_source.get("return_price_field", "close")) series = price_snapshot.get("series") if not isinstance(series, dict): report["reason"] = "price_snapshot_invalid" return run_store.persist(report) events: list[dict[str, Any]] = [] try: for entry in vintages: snapshot = validated_vintage_snapshots[str(entry["vintage_id"])] result = compute_tourism_signal(snapshot) events.append({ "event_id": entry["vintage_id"], "published_at": snapshot["source"]["published_at"], "signals": result["signals"], }) price_rows = {symbol: _series_rows(value, price_field, symbol) for symbol, value in series.items() if symbol != price_snapshot.get("benchmark_symbol")} benchmark_symbol = price_snapshot.get("benchmark_symbol") benchmark_rows = _series_rows(series[benchmark_symbol], price_field, str(benchmark_symbol)) if benchmark_symbol else None result = run_event_study( events, price_rows, benchmark_prices=benchmark_rows, windows=windows, cost_bps=cost_bps, min_events=min_events, execution_lag_sessions=execution_lag_sessions, ) except (FileNotFoundError, VintageStoreError, EventStudyError, ResearchRunError, KeyError, TypeError, ValueError) as exc: report["reason"] = "research_inputs_invalid" report["error"] = str(exc) return run_store.persist(report) report["status"] = "ready" report["reason"] = None report["result"] = result report["input_summary"] = { "event_count": len(events), "price_symbols": sorted(price_rows), "benchmark_symbol": benchmark_symbol, "return_price_field": price_field, } return run_store.persist(report)