set50-system/backend/app/forward_test.py

"""Forward-test paper-portfolio lifecycle with frozen signals (real, not cosmetic).

The earlier "forward" mode was cosmetic: it ran the same single-pass
backtest as "backtest", differing only in the `mode` string. This module
implements a genuine forward lifecycle:

    1. CREATE  (freeze):  capture the per-symbol combined scores / dividend
                          flags **at a specific as_of** and persist them. The
                          signal set is immutable from this point — later data
                          cannot change what the run decided.
    2. EXECUTE (paper):    use prices known *after* the freeze date to fill
                          the 50/20/30 dividend buckets (the engine does NOT
                          reach back to pick trades, it fills what the frozen
                          signal said).
    3. MARK    (mark-to-market): recompute portfolio equity against later
                          prices, recording an equity series.

Each step is persisted to a durable store (JSON, atomic write) so runs
survive restarts — replacing the in-memory backtest_runs list.

Honesty: a frozen forward run is paper and labelled as such. It uses the
score source supplied at CREATE time; if that source is the current board it
is non-PIT and the run is tagged `non_pit=true`.
"""

from __future__ import annotations

import json
import datetime as dt
import math
import threading
from copy import deepcopy
from pathlib import Path
from typing import Any, Optional
from uuid import uuid4

FORWARD_SCHEMA_VERSION = 1

# lifecycle statuses in order
_ORDER = ["frozen", "executed", "marked", "matured"]


class ForwardError(ValueError):
    pass


def _now() -> str:
    return dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds")


class ForwardTestStore:
    """Durable, thread-safe JSON store of forward-test runs."""

    def __init__(self, path: Path | str | None = None) -> None:
        self.path = Path(path).resolve() if path else None
        self._runs: dict[str, dict] = {}
        self._lock = threading.Lock()
        if self.path and self.path.is_file():
            self._load()

    # -- persistence ------------------------------------------------------
    def _load(self) -> None:
        try:
            payload = json.loads(self.path.read_text(encoding="utf-8"))
        except (OSError, json.JSONDecodeError) as exc:
            raise ForwardError(f"forward store unreadable: {exc}") from exc
        if not isinstance(payload, dict) or payload.get("schema_version") != FORWARD_SCHEMA_VERSION:
            raise ForwardError("unsupported forward store schema")
        self._runs = deepcopy(payload.get("runs", {}))

    def _persist(self) -> None:
        if not self.path:
            return
        self.path.parent.mkdir(parents=True, exist_ok=True)
        payload = {"schema_version": FORWARD_SCHEMA_VERSION, "runs": self._runs}
        tmp = self.path.with_name(f".{self.path.name}.tmp")
        tmp.write_text(json.dumps(payload, ensure_ascii=False, sort_keys=True) + "\n", encoding="utf-8")
        tmp.replace(self.path)

    # -- lifecycle --------------------------------------------------------
    def create(self, capital: float, frozen: dict[str, dict], as_of: str,
               non_pit: bool, note: str = "") -> dict:
        """CREATE: persist an immutable frozen-signal run."""
        with self._lock:
            if capital <= 0:
                raise ForwardError("capital must be > 0")
            run_id = f"fwd_{uuid4().hex[:12]}"
            run = {
                "id": run_id,
                "created_at": _now(),
                "capital": capital,
                "status": "frozen",
                "as_of": as_of,
                "non_pit": bool(non_pit),
                "note": note,
                "frozen_signals": deepcopy(frozen),
                "execution_prices": {},
                "holdings": {},
                "invested": 0.0,
                "equity_history": [{"date": as_of, "equity": capital}],
            }
            self._runs[run_id] = run
            self._persist()
            return deepcopy(run)

    def execute(self, run_id: str, execution_prices: dict[str, float]) -> dict:
        """EXECUTE: fill the frozen signals at prices provided (post-freeze).

        Uses the canonical 50/20/30 dividend allocation over the frozen
        combined scores (mirrors simulation.allocate_capital). Frozen signals
        are NOT re-readable/changeable here — only execution prices vary.
        """
        with self._lock:
            run = self._runs.get(run_id)
            if run is None:
                raise ForwardError(f"unknown forward run: {run_id}")
            if run["status"] != "frozen":
                raise ForwardError("run already executed")
            from . import simulation as sim
            candidates = []
            for sym, meta in (run.get("frozen_signals") or {}).items():
                price = execution_prices.get(sym)
                if price is None or price <= 0:
                    continue
                candidates.append(sim.Candidate(
                    symbol=sym, price=price,
                    combined_score=float(meta.get("combined", 0.0)),
                    is_dividend=bool(meta.get("is_dividend")),
                    dividend_yield=float(meta.get("dividend_yield") or 0.0),
                ))
            alloc = sim.allocate_capital(run["capital"], candidates)
            holdings = {o.symbol: o.qty for o in alloc.orders}
            invested = alloc.invested
            run["status"] = "executed"
            run["execution_prices"] = {s: float(execution_prices.get(s, 0)) for s in holdings}
            run["holdings"] = holdings
            run["invested"] = invested
            run["equity_history"].append({"date": _now(), "equity": invested})
            self._persist()
            return deepcopy(run)

    def mark(self, run_id: str, prices: dict[str, float]) -> dict:
        """MARK: recompute equity at provided (latest) prices and append."""
        with self._lock:
            run = self._runs.get(run_id)
            if run is None:
                raise ForwardError(f"unknown forward run: {run_id}")
            if run["status"] not in ("executed", "marked"):
                raise ForwardError("run not executed yet")
            equity = run["capital"]
            for sym, qty in (run.get("holdings") or {}).items():
                px = prices.get(sym)
                if px is None:
                    continue
                equity += qty * (px - run["execution_prices"].get(sym, 0))
            run["status"] = "marked"
            run["equity_history"].append({"date": _now(), "equity": round(equity, 2)})
            self._persist()
            return deepcopy(run)

    def mature(self, run_id: str, final_prices: dict[str, float]) -> dict:
        """MATURE: close the run at final prices, freeze the outcome."""
        with self._lock:
            run = self._runs.get(run_id)
            if run is None:
                raise ForwardError(f"unknown forward run: {run_id}")
            if run["status"] not in ("executed", "marked"):
                raise ForwardError("run cannot mature before execution")
            equity = run["capital"]
            for sym, qty in (run.get("holdings") or {}).items():
                px = final_prices.get(sym)
                if px is None:
                    continue
                equity += qty * (px - run["execution_prices"].get(sym, 0))
            run["status"] = "matured"
            run["final_equity"] = round(equity, 2)
            run["net_return"] = round((equity - run["capital"]) / run["capital"], 4) if run["capital"] else 0.0
            run["equity_history"].append({"date": _now(), "equity": round(equity, 2)})
            self._persist()
            return deepcopy(run)

    # -- reads ------------------------------------------------------------
    def get(self, run_id: str) -> Optional[dict]:
        run = self._runs.get(run_id)
        return deepcopy(run) if run else None

    def list(self) -> list[dict]:
        return [deepcopy(run) for run in sorted(self._runs.values(), key=lambda r: r.get("created_at", ""))]