"""Forward-test paper-portfolio lifecycle with frozen signals (real, not cosmetic). The earlier "forward" mode was cosmetic: it ran the same single-pass backtest as "backtest", differing only in the `mode` string. This module implements a genuine forward lifecycle: 1. CREATE (freeze): capture the per-symbol combined scores / dividend flags **at a specific as_of** and persist them. The signal set is immutable from this point — later data cannot change what the run decided. 2. EXECUTE (paper): use prices known *after* the freeze date to fill the 50/20/30 dividend buckets (the engine does NOT reach back to pick trades, it fills what the frozen signal said). 3. MARK (mark-to-market): recompute portfolio equity against later prices, recording an equity series. Each step is persisted to a durable store (JSON, atomic write) so runs survive restarts — replacing the in-memory backtest_runs list. Honesty: a frozen forward run is paper and labelled as such. It uses the score source supplied at CREATE time; if that source is the current board it is non-PIT and the run is tagged `non_pit=true`. """ from __future__ import annotations import json import datetime as dt import math import threading from copy import deepcopy from pathlib import Path from typing import Any, Optional from uuid import uuid4 FORWARD_SCHEMA_VERSION = 1 # lifecycle statuses in order _ORDER = ["frozen", "executed", "marked", "matured"] class ForwardError(ValueError): pass def _now() -> str: return dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds") class ForwardTestStore: """Durable, thread-safe JSON store of forward-test runs.""" def __init__(self, path: Path | str | None = None) -> None: self.path = Path(path).resolve() if path else None self._runs: dict[str, dict] = {} self._lock = threading.Lock() if self.path and self.path.is_file(): self._load() # -- persistence ------------------------------------------------------ def _load(self) -> None: try: payload = json.loads(self.path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise ForwardError(f"forward store unreadable: {exc}") from exc if not isinstance(payload, dict) or payload.get("schema_version") != FORWARD_SCHEMA_VERSION: raise ForwardError("unsupported forward store schema") self._runs = deepcopy(payload.get("runs", {})) def _persist(self) -> None: if not self.path: return self.path.parent.mkdir(parents=True, exist_ok=True) payload = {"schema_version": FORWARD_SCHEMA_VERSION, "runs": self._runs} tmp = self.path.with_name(f".{self.path.name}.tmp") tmp.write_text(json.dumps(payload, ensure_ascii=False, sort_keys=True) + "\n", encoding="utf-8") tmp.replace(self.path) # -- lifecycle -------------------------------------------------------- def create(self, capital: float, frozen: dict[str, dict], as_of: str, non_pit: bool, note: str = "") -> dict: """CREATE: persist an immutable frozen-signal run.""" with self._lock: if capital <= 0: raise ForwardError("capital must be > 0") run_id = f"fwd_{uuid4().hex[:12]}" run = { "id": run_id, "created_at": _now(), "capital": capital, "status": "frozen", "as_of": as_of, "non_pit": bool(non_pit), "note": note, "frozen_signals": deepcopy(frozen), "execution_prices": {}, "holdings": {}, "invested": 0.0, "equity_history": [{"date": as_of, "equity": capital}], } self._runs[run_id] = run self._persist() return deepcopy(run) def execute(self, run_id: str, execution_prices: dict[str, float]) -> dict: """EXECUTE: fill the frozen signals at prices provided (post-freeze). Uses the canonical 50/20/30 dividend allocation over the frozen combined scores (mirrors simulation.allocate_capital). Frozen signals are NOT re-readable/changeable here — only execution prices vary. """ with self._lock: run = self._runs.get(run_id) if run is None: raise ForwardError(f"unknown forward run: {run_id}") if run["status"] != "frozen": raise ForwardError("run already executed") from . import simulation as sim candidates = [] for sym, meta in (run.get("frozen_signals") or {}).items(): price = execution_prices.get(sym) if price is None or price <= 0: continue candidates.append(sim.Candidate( symbol=sym, price=price, combined_score=float(meta.get("combined", 0.0)), is_dividend=bool(meta.get("is_dividend")), dividend_yield=float(meta.get("dividend_yield") or 0.0), )) alloc = sim.allocate_capital(run["capital"], candidates) holdings = {o.symbol: o.qty for o in alloc.orders} invested = alloc.invested run["status"] = "executed" run["execution_prices"] = {s: float(execution_prices.get(s, 0)) for s in holdings} run["holdings"] = holdings run["invested"] = invested run["equity_history"].append({"date": _now(), "equity": invested}) self._persist() return deepcopy(run) def mark(self, run_id: str, prices: dict[str, float]) -> dict: """MARK: recompute equity at provided (latest) prices and append.""" with self._lock: run = self._runs.get(run_id) if run is None: raise ForwardError(f"unknown forward run: {run_id}") if run["status"] not in ("executed", "marked"): raise ForwardError("run not executed yet") equity = run["capital"] for sym, qty in (run.get("holdings") or {}).items(): px = prices.get(sym) if px is None: continue equity += qty * (px - run["execution_prices"].get(sym, 0)) run["status"] = "marked" run["equity_history"].append({"date": _now(), "equity": round(equity, 2)}) self._persist() return deepcopy(run) def mature(self, run_id: str, final_prices: dict[str, float]) -> dict: """MATURE: close the run at final prices, freeze the outcome.""" with self._lock: run = self._runs.get(run_id) if run is None: raise ForwardError(f"unknown forward run: {run_id}") if run["status"] not in ("executed", "marked"): raise ForwardError("run cannot mature before execution") equity = run["capital"] for sym, qty in (run.get("holdings") or {}).items(): px = final_prices.get(sym) if px is None: continue equity += qty * (px - run["execution_prices"].get(sym, 0)) run["status"] = "matured" run["final_equity"] = round(equity, 2) run["net_return"] = round((equity - run["capital"]) / run["capital"], 4) if run["capital"] else 0.0 run["equity_history"].append({"date": _now(), "equity": round(equity, 2)}) self._persist() return deepcopy(run) # -- reads ------------------------------------------------------------ def get(self, run_id: str) -> Optional[dict]: run = self._runs.get(run_id) return deepcopy(run) if run else None def list(self) -> list[dict]: return [deepcopy(run) for run in sorted(self._runs.values(), key=lambda r: r.get("created_at", ""))]