diff --git a/backend/app/backtest_engine.py b/backend/app/backtest_engine.py index 5df8204..e4216a0 100644 --- a/backend/app/backtest_engine.py +++ b/backend/app/backtest_engine.py @@ -194,11 +194,16 @@ def run_event_backtest( nonlocal last_scores, leakage_guard as_of = sig.released_at syms = symbols or list(price_series.keys()) - if score_fn is not None: - sc = score_fn(syms, as_of) or {} - else: - from .dashboard import default_scores - sc = default_scores(syms) or {} + if score_fn is None: + # Strict event-driven mode must never fall back to the live board: + # default_scores() builds the CURRENT board with no as_of handling, + # so freezing it at a historical release would be look-ahead even + # though it does not set leakage_guard. Fail closed instead. + raise ValueError( + "strict event-driven backtest requires a PIT score_fn; " + "refusing to fall back to the live (non-PIT) board" + ) + sc = score_fn(syms, as_of) or {} # leakage_guard only when the scores attest PIT provenance any_pit = any( isinstance(m, dict) and isinstance(m.get("pit_meta"), dict) diff --git a/backend/scripts/probe_event_backtest.py b/backend/scripts/probe_event_backtest.py new file mode 100644 index 0000000..38c227c --- /dev/null +++ b/backend/scripts/probe_event_backtest.py @@ -0,0 +1,89 @@ +"""Live end-to-end probe of the event-driven PIT backtest engine (Task 8). + +Uses fully PIT-complete fake inputs to prove the engine produces a reconciled, +accounting-consistent result — this is the "happy path" complement to the +fail-closed readiness gate already verified separately. +""" +import datetime as dt +from app.backtest_engine import run_event_backtest +from app.backtest_readiness import _factor_keys_required + + +def series(syms, days=120, jump=None, after_price=20.0): + s = dt.date(2026, 1, 1) + out = {} + for sym in syms: + bars = [] + for i in range(days): + d = (s + dt.timedelta(days=i)) + px = (after_price if jump and d >= jump else 10.0) + bars.append({"date": d.isoformat(), "adjusted_close": px}) + out[sym] = {"bars": bars} + return out + + +class FStore: + def __init__(self, rels): + self.r = rels + + def series(self, k): + return [{"released_at": t, "observed_at": t, "value": 1.0} + for t in self.r.get(k, [])] + + +class FLedger: + def symbols(self): + return ["A"] + + def entries(self, s): + return [{"symbol": "A", "ex_date": "2026-02-15", + "per_share": 1.0, "estimate": False}] + + +rels = {k: ["2026-01-02T09:00:00+07:00"] for k in _factor_keys_required()} + + +class SC: + def list_ids(self): + return ["1"] + + def _load_manifest(self): + return {"snapshots": {"1": {"retrieved_at": "2026-01-02T09:00:00+07:00"}}} + + +def pit_scorer(symbols, as_of=None): + out = {} + for i, sym in enumerate(symbols): + out[sym] = { + "combined": 1.0 / (i + 1), "is_dividend": True, + "dividend_yield": 3.0, + "pit_meta": {"pit": True, "partial_pit": False, "note": "test"}, + } + return out + + +ser = series(["A", "B", "C"], 120, jump=dt.date(2026, 2, 1)) +res = run_event_backtest( + start="2026-01-05", end="2026-04-30", capital=100000, + factor_store=FStore(rels), siamchart_store=SC(), + dividend_ledger=FLedger(), price_series=ser, + score_fn=pit_scorer, symbols=["A", "B", "C"], +) +d = res.to_dict() +print("rebalances:", d["rebalances"]) +print("leakage_guard:", d["leakage_guard"]) +print("accounting_reconciled:", d["accounting_reconciled"]) +print("final_equity:", round(d["final_equity"], 2)) +print("realized:", d["realized_trading_pnl"], "unrealized:", d["unrealized_trading_pnl"]) +print("dividend_cash:", d["dividend_cash_received"], "dividend_recv:", d["dividend_receivable"]) +print("fees:", d["transaction_costs"], "net_return:", round(d["net_return"], 4)) +print("holdings:", [(h["symbol"], h["qty"]) for h in d["holdings"]]) + +assert d["accounting_reconciled"], "FAIL: not reconciled" +lhs = d["final_equity"] - 100000 +rhs = (d["realized_trading_pnl"] + d["unrealized_trading_pnl"] + + d["dividend_cash_received"] - d["transaction_costs"]) +print(f" identity lhs={round(lhs,2)} rhs={round(rhs,2)}") +assert abs(lhs - rhs) < 1.0, "identity mismatch" +assert d["leakage_guard"] is True, "leakage_guard should be True with pit_meta" +print("PASS") diff --git a/backend/tests/test_backtest_engine.py b/backend/tests/test_backtest_engine.py index 25e8d80..34a49d8 100644 --- a/backend/tests/test_backtest_engine.py +++ b/backend/tests/test_backtest_engine.py @@ -160,6 +160,17 @@ class ResultContractTest(unittest.TestCase): self.assertEqual(d["fee_rate"], 0.003) self.assertEqual(d["dividend_timing"], "ex_date_plus_30d") + def test_refuses_to_fallback_to_live_board_without_scorer(self): + # strict event-driven mode must not freeze the live (non-PIT) board at a + # historical release — that is look-ahead. It must fail closed. + series = make_series(["A"], "2026-01-01", 120) + with self.assertRaises(ValueError): + run_event_backtest( + start="2026-01-01", end="2026-04-30", + capital=100_000, price_series=series, + symbols=["A"], # no score_fn -> must raise + ) + if __name__ == "__main__": unittest.main() diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 381a156..00bfec2 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -157,6 +157,47 @@ NEVER include API keys, tokens, passwords, secrets, credentials, or connection s --- +## Session 2026-08-28 — Event-driven PIT backtest (verified + pushed) + +**Branch:** `main` · **HEAD:** `d73a58b` (+cycle-2 fix pending) · **Pushed:** yes (tasks 1–7) + +### Goal implemented +Replace the calendar-rebalance backtest with a strict point-in-time, event-driven +engine per the user's description. Confirmed decisions: +1. **Strict PIT** — block when factor / Siamchart / price coverage is incomplete (no fallback). +2. **Execution** — freeze signal at release date D, execute at next trading-day close after D. +3. **Dividend timing** — entitled on ex-date; cash available `ex_date + 30` days (`ex_date_plus_30d`). +4. **Average cost** — realized P&L uses weighted avg cost. +5. **Fee** — all-in 0.3% per trade, no added VAT. + +### Shipped (commits on main) +- `68f2cc1` Task 1 `backtest_readiness.py` — readiness + recommended start/end defaults, fail-closed. +- `6439e9c` Task 2 `backtest_events.py` — event calendar + next-trading-day mapping. +- `dc057e9` Task 3 `portfolio_ledger.py` — cash/positions/avg-cost/realized+unrealized P&L/dividend receivables/fees. +- `71b893e` Task 4 `portfolio_rebalancer.py` — 50/20/30 → 100-lot executable orders, sells-first, no trade on unchanged target. +- `6459015` Task 5 `backtest_engine.py` — event processor; refuses live-board fallback without a PIT scorer. +- `c87767c` Task 6 `backtest_store.py` + strict `POST /api/v1/backtest/run` (readiness-gated) + `GET /api/v1/backtest/run(+)`. +- `d73a58b` Task 7 frontend — readiness gate disables Run + shows missing coverage; detailed report. + +### Verification +- Standalone offline suite **52 tests** (readiness 9, events 14, ledger 15, rebalancer 5, engine 5, store 4). compileall + diff-check clean. +- Static scan: no secrets/eval/debug (4 "secret" regex hits are the prose "missing token" string). +- Live readiness on real data: `ready=false`, missing factor+siamchart, recommended_end=2026-08-27. +- Live strict run on real data (no vintages): `POST /api/v1/backtest/run` → **400** + missing list (fail-closed). +- Happy-path `scripts/probe_event_backtest.py` (synthetic PIT-complete): accounting_reconciled=True, identity lhs=rhs, leakage_guard=True, 100-lot holdings. +- Cycle-1 independent review `deleg_5225e9de` flagged: engine's no-scorer fallback used the live board (look-ahead). Fixed to fail-closed. Cycle-2 review `deleg_b969075c` re-verifying. +- Evidence: `docs/test-evidence/2026-08-28-event-driven-backtest.md`. + +### Honest scope / limitations +- Current on-disk data has **no factor vintages / Siamchart manifest** → strict event backtest is blocked on real data until collection accumulates (this is the intended fail-closed behavior, matching "strict PIT or block"). +- Dividend payment timing is the confirmed `ex_date + 30d` **assumption** (Siamchart gives ex-date+DPS, not pay date); every result exposes `dividend_timing=ex_date_plus_30d`. +- Only the new `/api/v1/backtest/run` route is event-driven/strict; the legacy `POST /api/v1/backtest` remains as the descriptive non-PIT path. + +## Exact next action +Collect factor vintages + Siamchart snapshots over time (scheduler already seeds Siamchart baseline) until `/api/v1/backtest/readiness` reports `ready=true`, then run a real strict event backtest. Frontend already wires readiness → recommended dates. + +--- + ## Session 2026-08-27 — Backtest Accounting Remediation **Branch:** `main` · **Base HEAD:** `b362cc3` · **Commit/push:** not performed diff --git a/docs/engineering-log.md b/docs/engineering-log.md index faaf6de..9b94260 100644 --- a/docs/engineering-log.md +++ b/docs/engineering-log.md @@ -106,3 +106,4 @@ - Real dated dividend-history collector (2026-08-27, commit `f9973e8`): `siamchart.py` gained `parse_dividend_history`/`fetch_dividend_history` (reads the "ประวัติการปันผล" ex-date + DPS table per stock-info page); `dividend_ledger.py` gained `populate_dated_dividends` (registers dated rows, estimate=False). `POST /api/v1/dividends/update` fetches all snapshot symbols and persists a dated ledger at `data/dividends/ledger.json`; `use_ledger` backtests prefer the dated ledger (`dividend_method=dated_ledger`) and fall back to DPS estimates when unpopulated. Full backend **292 passed** (was 286). Live network fetch: 49/49 symbols, 1410 dated payments; use_ledger → `dated_ledger`. Note: this sandbox HAS outbound network (curl/https to siamchart 200) — contrary to earlier assumption. - Auto-refresh dated dividend ledger (2026-08-27, commit `03195dc`): `AppDataScheduler` now calls `_maybe_refresh_dated_dividends()` in `refresh_all` — a cooldown-gated (default 6h) fetch of real dated dividend history into `data/dividends/ledger.json`, so the ledger stays fresh without manual `/api/v1/dividends/update` and without hammering the source every tick. `DIVIDEND_REFRESH_COOLDOWN_SECONDS` env toggle. Full backend **294 passed** (was 292). - Single-container Docker packaging (2026-08-27, commit `f9b8cd1`): added `Dockerfile` (python:3.11-slim + nginx; serves the PREBUILT `frontend/dist` SPA, proxies `/api` → 127.0.0.1:5000, `VOLUME /app/backend/data`, HEALTHCHECK on /api/v1/dashboard/summary, exposes :80), `deploy/nginx.conf`, `deploy/entrypoint.sh` (Flask HOST=0.0.0.0:5000 + nginx foreground), `docker-compose.yml` (port 8080:80, volume `./backend/data`), `.dockerignore`, and `.gitignore` now tracks `frontend/dist/` (prebuilt bundle required by the image; `backend/data/` stays untracked). Backend runtime verified (test_client `/api/v1/dashboard/summary` = 200). No local docker on this machine so the image itself must build on EasyPanel. +- Event-driven PIT backtest (2026-08-28, commits `68f2cc1` → `d73a58b`, 8-phase plan at `.hermes/plans/2026-08-28_091000-event-driven-pit-backtest.md`): replaces the calendar-rebalance backtest with a strict point-in-time event-driven engine. Confirmed user decisions: (1) strict PIT — block when factor/Siamchart/price coverage is incomplete; (2) execute at next trading-day close after release (no same-day look-ahead); (3) dividend cash available ex_date+30 calendar days (`ex_date_plus_30d` assumption, not an observed pay date — Siamchart source gives ex-date+DPS only); (4) average-cost realized P&L; (5) all-in 0.3% fee per trade, no added tax. Ships: `backtest_readiness.py` (readiness + recommended start/end defaults, fail-closed), `backtest_events.py` (event calendar + next-trading-day mapping), `portfolio_ledger.py` (cash/positions/avg-cost/realized+unrealized P&L/dividend receivables/fees; gross realized + fees subtracted once), `portfolio_rebalancer.py` (50/20/30 → executable 100-lot orders, sells-first, no trade on unchanged target), `backtest_engine.py` (event processor; refuses to fall back to the live board without a PIT scorer — fail-closed, was flagged by cycle-1 review `deleg_5225e9de` and fixed), `backtest_store.py` (durable atomic JSON runs at `data/backtest/runs.json`), strict route `POST /api/v1/backtest/run` (readiness-gate → 400 with missing list when not ready) + `GET /api/v1/backtest/run(+)`, and frontend (readiness gate disables Run + shows missing coverage; report shows realized/unrealized P&L, dividend received/receivable, fees, final holdings with avg cost/last price/unrealized). Offline standalone suite **52 tests** (9+14+15+5+5+4); compileall + diff-check clean; static scan no secrets/eval/debug (4 "secret"-regex hits are the prose "missing token" string, not credentials). Live probes: `GET /api/v1/backtest/readiness` on real data → ready=false, recommended_end=2026-08-27; `POST /api/v1/backtest/run` on real data (no vintages) → 400 + missing list; happy-path `scripts/probe_event_backtest.py` (synthetic PIT-complete) → accounting_reconciled=True, identity lhs=rhs, leakage_guard=True, 100-lot holdings. diff --git a/docs/test-evidence/2026-08-28-event-driven-backtest.md b/docs/test-evidence/2026-08-28-event-driven-backtest.md new file mode 100644 index 0000000..e6b6b87 --- /dev/null +++ b/docs/test-evidence/2026-08-28-event-driven-backtest.md @@ -0,0 +1,89 @@ +# Test Evidence — Event-Driven PIT Backtest (2026-08-28) + +## Summary + +Replaced the calendar-based backtest loop with a strict point-in-time, +event-driven engine implementing the user-confirmed lifecycle. + +## Confirmed user decisions implemented + +1. **Strict PIT** — backtest blocks (400 + missing list) when factor vintages, + Siamchart manifest, or executable prices do not fully cover the window. +2. **Execution timing** — signal frozen on release date D; executed at next + trading-day close after D (no same-day look-ahead). +3. **Dividend timing** — entitled on ex-date (shares held before it); cash + available exactly `ex_date + 30` calendar days, exposed as + `ex_date_plus_30d`. +4. **Average cost** — realized P&L uses weighted average cost. +5. **Fees** — all-in 0.3% of notional on every buy and every sell; no added VAT. + +## Tasks shipped (commits on main) + +| Task | Commit | Tests | +|---|---|---| +| 1. PIT readiness + default dates | `68f2cc1` | 9 | +| 2. Unified event calendar | `6439e9c` | 14 | +| 3. Portfolio accounting ledger | `dc057e9` | 15 | +| 4. Lot/cash-constrained rebalancer | `71b893e` | 5 | +| 5. Event-driven backtest engine | `6459015` | 4 | +| 6. Durable run store + strict route | `c87767c` | 4 | +| 7. Frontend readiness/report | `d73a58b` | build | +| 8. Verification/review | (this doc) | — | + +## Standalone suite (unittest, offline, deterministic) + +``` +51 tests OK + test_backtest_readiness (9) test_backtest_events (14) + test_portfolio_ledger (15) test_portfolio_rebalancer (5) + test_backtest_engine (4) test_backtest_store (4) +``` + +Compileall (all new modules): pass. `git diff --check`: clean. + +## Static security scan (new backend files) + +- secrets regex: 4 hits in `backtest_readiness.py` — all are the prose string + "missing token" (the umbrella missing-coverage token), **not** credentials. +- eval/exec: 0. Debug/print: 0. +- No hardcoded credentials, no path traversal, no unsafe eval. + +## Live probes + +### Fail-closed readiness (real local data — no factor vintages yet) + +`POST /api/v1/backtest/run` on the on-disk data (0 factor vintages, no Siamchart +manifest) returns **400** with `missing` listing factor keys + siamchart — it +does not manufacture a fake PIT window. + +`GET /api/v1/backtest/readiness` on real data: `ready=false`, +`missing` includes factor + siamchart, `recommended_end=2026-08-27` (yesterday, +Bangkok). + +### Happy-path engine (synthetic PIT-complete inputs) + +`backend/scripts/probe_event_backtest.py` (deterministic): +- `accounting_reconciled: True` +- accounting identity satisfied: `final_equity - capital == realized + unrealized + dividend_cash - fees` + (`lhs=84760.0 == rhs=84760.0`) +- `leakage_guard: True` (pit_meta-attested scorer) +- dividend cash credited (ex_date+30), receivable 0 by end +- holdings all 100-lot multiples (A=5000, B=3000) +- fee 0.3% applied (240.0) + +## Accounting invariant + +``` +ending_equity - initial_capital + = realized_trading_pnl (gross, avg cost) + + unrealized_trading_pnl + + dividend_cash_received + - transaction_costs +``` + +`price_pnl = realized + unrealized` (compat). + +## Independent review + +See the reviewer subagent verdict recorded in the commit / engineering-log for +this session (`deleg_5225e9de`).