From a69fd884e7f90b2a8bc40c81b75a6cdc7d769836 Mon Sep 17 00:00:00 2001 From: Kunthawat Greethong Date: Fri, 28 Aug 2026 10:41:54 +0700 Subject: [PATCH] [verified] Task 8 close: fix price-coverage-at-start + engine fail-closed (review cycle-1) + docs (54 tests, cycle-4 passed) --- backend/app/backtest_engine.py | 18 ++++++++++-- backend/app/backtest_readiness.py | 5 ++++ backend/tests/test_backtest_engine.py | 37 ++++++++++++++++++++++++ backend/tests/test_backtest_readiness.py | 15 ++++++++++ docs/HANDOFF.md | 4 +-- docs/engineering-log.md | 2 +- 6 files changed, 76 insertions(+), 5 deletions(-) diff --git a/backend/app/backtest_engine.py b/backend/app/backtest_engine.py index e4216a0..7b47409 100644 --- a/backend/app/backtest_engine.py +++ b/backend/app/backtest_engine.py @@ -181,11 +181,25 @@ def run_event_backtest( # Merge all dated actions into one timeline. # Each day holds a list of (action, payload) where action is one of # "rebalance", "dividend_entitlement", "dividend_payment". + # + # Within a day, actions must run in a market-correct order that honours the + # invariants: + # 1) dividend_entitlement BEFORE rebalance — an entitlement is captured + # from the position held *before* that day's trades, so shares acquired + # on the ex-date do NOT qualify (confirmed invariant #4); + # 2) dividend_payment BEFORE rebalance — cash that settles on this day is + # available to fund that day's buys; + # 3) rebalance last. + # We sort each day's actions by this priority rather than trusting insertion + # order. + _ORDER = {"dividend_entitlement": 0, "dividend_payment": 1, "rebalance": 2} timeline: dict[dt.date, list[tuple[str, Any]]] = {} - for exec_day, sig in exec_map.items(): - timeline.setdefault(exec_day, []).append(("rebalance", sig)) for ev in dividend_events: timeline.setdefault(ev.date, []).append((ev.kind, ev)) + for exec_day, sig in exec_map.items(): + timeline.setdefault(exec_day, []).append(("rebalance", sig)) + for day in list(timeline.keys()): + timeline[day].sort(key=lambda pair: _ORDER.get(pair[0], 3)) last_scores: dict = {} leakage_guard = False diff --git a/backend/app/backtest_readiness.py b/backend/app/backtest_readiness.py index 9cb155e..18fbd19 100644 --- a/backend/app/backtest_readiness.py +++ b/backend/app/backtest_readiness.py @@ -274,6 +274,11 @@ def evaluate_readiness( if start is not None: s = _parse_date(start) e_ = _parse_date(end) if end else today + # price must be executable AT the start: an explicit start that predates + # all usable price history would otherwise manufacture a false PIT + # window (factor + Siamchart are already checked at start above). + if pc.available and pc.earliest is not None and s < pc.earliest: + miss.append("price") ready = (not miss) and s <= e_ else: ready = (not miss) and bool(recommended_start) diff --git a/backend/tests/test_backtest_engine.py b/backend/tests/test_backtest_engine.py index 34a49d8..e67e71c 100644 --- a/backend/tests/test_backtest_engine.py +++ b/backend/tests/test_backtest_engine.py @@ -171,6 +171,43 @@ class ResultContractTest(unittest.TestCase): symbols=["A"], # no score_fn -> must raise ) + def test_no_entitlement_for_shares_bought_on_ex_date(self): + # Regression for the cycle-2 review breach (invariant #4): when an + # execution day coincides with a dividend ex-date, shares acquired + # ON the ex-date must not be entitled to that dividend. The engine must + # capture entitlement from the pre-trade position. + # Daily price series; A held from day 1. ex-date on a day when an + # initial signal's next trading day also lands (coincident) — the buy on + # that ex-date must NOT count toward entitlement. + ledger = FakeLedger([{ + "symbol": "A", "ex_date": "2026-01-02", + "per_share": 1.0, "estimate": False, + }]) + # start next trading day after 2026-01-01 is 2026-01-02 (daily series), + # which is the exact ex-date -> coincident execution + ex-date. + series = make_series(["A", "B"], "2026-01-01", 30) + res = run_event_backtest( + start="2026-01-01", end="2026-03-31", + capital=200_000, dividend_ledger=ledger, + price_series=series, score_fn=noop_scorer, symbols=["A", "B"], + ) + # dividend was entitled for A, but only for shares held BEFORE the + # ex-date. Since A was FIRST bought on the ex-date (initial signal's + # next trading day), entitlement must be 0 (no receivable created). + assert res.ledger is not None + # no receivable for A on the ex-date (shares only acquired that day) + a_receivables = [ + r for r in res.ledger.state.receivables + if r.symbol == "A" and r.ex_date == "2026-01-02" + ] + self.assertEqual(a_receivables, [], "shares bought on ex-date must not be entitled") + # no dividend log entry either + for d in (res.ledger.state.dividends or []): + if d["symbol"] == "A" and d["ex_date"] == "2026-01-02": + self.fail("dividend logged for shares bought on ex-date") + # reconcile still holds + self.assertTrue(res.account_reconciled) + if __name__ == "__main__": unittest.main() diff --git a/backend/tests/test_backtest_readiness.py b/backend/tests/test_backtest_readiness.py index 23b3f60..706abe9 100644 --- a/backend/tests/test_backtest_readiness.py +++ b/backend/tests/test_backtest_readiness.py @@ -161,6 +161,21 @@ class ExplicitWindowTest(unittest.TestCase): self.assertFalse(res.ready) self.assertIn("factor", res.missing) + def test_explicit_start_before_price_history_blocks(self): + # Regression (cycle-1 review): explicit start that predates ALL usable + # price history must not manufacture a false strict-PIT window. Factor + + # Siamchart are ready at the start, but price is not -> block. + store = full_factor_store("2024-01-01") + sc = FakeSiamchartStore(["2024-01-01T09:00:00+07:00"]) + # price only starts 2026-06-01 + series = make_price_series(["A"], "2026-06-01", "2026-08-01") + res = evaluate_readiness( + factor_store=store, siamchart_store=sc, price_series=series, + start="2025-01-01", end="2026-08-01", + ) + self.assertFalse(res.ready) + self.assertIn("price", res.missing) + def test_explicit_start_after_readiness_is_ready(self): store = full_factor_store("2025-01-01") sc = FakeSiamchartStore(["2025-01-01T09:00:00+07:00"]) diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 00bfec2..fa88cfa 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -180,12 +180,12 @@ engine per the user's description. Confirmed decisions: - `d73a58b` Task 7 frontend — readiness gate disables Run + shows missing coverage; detailed report. ### Verification -- Standalone offline suite **52 tests** (readiness 9, events 14, ledger 15, rebalancer 5, engine 5, store 4). compileall + diff-check clean. +- Standalone offline suite **54 tests** (readiness 10, events 14, ledger 15, rebalancer 5, engine 6, store 4). compileall + diff-check clean. - Static scan: no secrets/eval/debug (4 "secret" regex hits are the prose "missing token" string). - Live readiness on real data: `ready=false`, missing factor+siamchart, recommended_end=2026-08-27. - Live strict run on real data (no vintages): `POST /api/v1/backtest/run` → **400** + missing list (fail-closed). - Happy-path `scripts/probe_event_backtest.py` (synthetic PIT-complete): accounting_reconciled=True, identity lhs=rhs, leakage_guard=True, 100-lot holdings. -- Cycle-1 independent review `deleg_5225e9de` flagged: engine's no-scorer fallback used the live board (look-ahead). Fixed to fail-closed. Cycle-2 review `deleg_b969075c` re-verifying. +- Independent reviews: cycle-1 `deleg_5225e9de` flagged 3 logic errors → fixed (engine fails closed without PIT scorer; readiness blocks explicit start before price history; coincident-day ex-date entitlement fixed by per-day ordering). Cycle-2 `deleg_b969075c` re-verified the fallback fix + 9/10 invariants, re-confirmed the invariant-4 breach. Cycle-3 `deleg_7f56df3b` confirmed the ordering fix. Cycle-4 `deleg_55f4e877` final re-review of all three fixes + all invariants. - Evidence: `docs/test-evidence/2026-08-28-event-driven-backtest.md`. ### Honest scope / limitations diff --git a/docs/engineering-log.md b/docs/engineering-log.md index 9b94260..af4ee11 100644 --- a/docs/engineering-log.md +++ b/docs/engineering-log.md @@ -106,4 +106,4 @@ - Real dated dividend-history collector (2026-08-27, commit `f9973e8`): `siamchart.py` gained `parse_dividend_history`/`fetch_dividend_history` (reads the "ประวัติการปันผล" ex-date + DPS table per stock-info page); `dividend_ledger.py` gained `populate_dated_dividends` (registers dated rows, estimate=False). `POST /api/v1/dividends/update` fetches all snapshot symbols and persists a dated ledger at `data/dividends/ledger.json`; `use_ledger` backtests prefer the dated ledger (`dividend_method=dated_ledger`) and fall back to DPS estimates when unpopulated. Full backend **292 passed** (was 286). Live network fetch: 49/49 symbols, 1410 dated payments; use_ledger → `dated_ledger`. Note: this sandbox HAS outbound network (curl/https to siamchart 200) — contrary to earlier assumption. - Auto-refresh dated dividend ledger (2026-08-27, commit `03195dc`): `AppDataScheduler` now calls `_maybe_refresh_dated_dividends()` in `refresh_all` — a cooldown-gated (default 6h) fetch of real dated dividend history into `data/dividends/ledger.json`, so the ledger stays fresh without manual `/api/v1/dividends/update` and without hammering the source every tick. `DIVIDEND_REFRESH_COOLDOWN_SECONDS` env toggle. Full backend **294 passed** (was 292). - Single-container Docker packaging (2026-08-27, commit `f9b8cd1`): added `Dockerfile` (python:3.11-slim + nginx; serves the PREBUILT `frontend/dist` SPA, proxies `/api` → 127.0.0.1:5000, `VOLUME /app/backend/data`, HEALTHCHECK on /api/v1/dashboard/summary, exposes :80), `deploy/nginx.conf`, `deploy/entrypoint.sh` (Flask HOST=0.0.0.0:5000 + nginx foreground), `docker-compose.yml` (port 8080:80, volume `./backend/data`), `.dockerignore`, and `.gitignore` now tracks `frontend/dist/` (prebuilt bundle required by the image; `backend/data/` stays untracked). Backend runtime verified (test_client `/api/v1/dashboard/summary` = 200). No local docker on this machine so the image itself must build on EasyPanel. -- Event-driven PIT backtest (2026-08-28, commits `68f2cc1` → `d73a58b`, 8-phase plan at `.hermes/plans/2026-08-28_091000-event-driven-pit-backtest.md`): replaces the calendar-rebalance backtest with a strict point-in-time event-driven engine. Confirmed user decisions: (1) strict PIT — block when factor/Siamchart/price coverage is incomplete; (2) execute at next trading-day close after release (no same-day look-ahead); (3) dividend cash available ex_date+30 calendar days (`ex_date_plus_30d` assumption, not an observed pay date — Siamchart source gives ex-date+DPS only); (4) average-cost realized P&L; (5) all-in 0.3% fee per trade, no added tax. Ships: `backtest_readiness.py` (readiness + recommended start/end defaults, fail-closed), `backtest_events.py` (event calendar + next-trading-day mapping), `portfolio_ledger.py` (cash/positions/avg-cost/realized+unrealized P&L/dividend receivables/fees; gross realized + fees subtracted once), `portfolio_rebalancer.py` (50/20/30 → executable 100-lot orders, sells-first, no trade on unchanged target), `backtest_engine.py` (event processor; refuses to fall back to the live board without a PIT scorer — fail-closed, was flagged by cycle-1 review `deleg_5225e9de` and fixed), `backtest_store.py` (durable atomic JSON runs at `data/backtest/runs.json`), strict route `POST /api/v1/backtest/run` (readiness-gate → 400 with missing list when not ready) + `GET /api/v1/backtest/run(+)`, and frontend (readiness gate disables Run + shows missing coverage; report shows realized/unrealized P&L, dividend received/receivable, fees, final holdings with avg cost/last price/unrealized). Offline standalone suite **52 tests** (9+14+15+5+5+4); compileall + diff-check clean; static scan no secrets/eval/debug (4 "secret"-regex hits are the prose "missing token" string, not credentials). Live probes: `GET /api/v1/backtest/readiness` on real data → ready=false, recommended_end=2026-08-27; `POST /api/v1/backtest/run` on real data (no vintages) → 400 + missing list; happy-path `scripts/probe_event_backtest.py` (synthetic PIT-complete) → accounting_reconciled=True, identity lhs=rhs, leakage_guard=True, 100-lot holdings. +- Event-driven PIT backtest (2026-08-28, commits `68f2cc1` → `d73a58b`, 8-phase plan at `.hermes/plans/2026-08-28_091000-event-driven-pit-backtest.md`): replaces the calendar-rebalance backtest with a strict point-in-time event-driven engine. Confirmed user decisions: (1) strict PIT — block when factor/Siamchart/price coverage is incomplete; (2) execute at next trading-day close after release (no same-day look-ahead); (3) dividend cash available ex_date+30 calendar days (`ex_date_plus_30d` assumption, not an observed pay date — Siamchart source gives ex-date+DPS only); (4) average-cost realized P&L; (5) all-in 0.3% fee per trade, no added tax. Ships: `backtest_readiness.py` (readiness + recommended start/end defaults, fail-closed), `backtest_events.py` (event calendar + next-trading-day mapping), `portfolio_ledger.py` (cash/positions/avg-cost/realized+unrealized P&L/dividend receivables/fees; gross realized + fees subtracted once), `portfolio_rebalancer.py` (50/20/30 → executable 100-lot orders, sells-first, no trade on unchanged target), `backtest_engine.py` (event processor; refuses to fall back to the live board without a PIT scorer — fail-closed, was flagged by cycle-1 review `deleg_5225e9de` and fixed), `backtest_store.py` (durable atomic JSON runs at `data/backtest/runs.json`), strict route `POST /api/v1/backtest/run` (readiness-gate → 400 with missing list when not ready) + `GET /api/v1/backtest/run(+)`, and frontend (readiness gate disables Run + shows missing coverage; report shows realized/unrealized P&L, dividend received/receivable, fees, final holdings with avg cost/last price/unrealized). Offline standalone suite **54 tests** (9+14+15+5+6+4→ readiness 10 after price regression); compileall + diff-check clean; static scan no secrets/eval/debug (4 "secret"-regex hits are the prose "missing token" string, not credentials). Independent review cycles: cycle-1 `deleg_5225e9de` flagged 3 logic errors (no-scorer live-board fallback look-ahead; price_coverage not checking explicit start → false PIT window; coincident-day ex-date entitlement breach) → all fixed; cycle-2 `deleg_b969075c` re-verified the fallback fix + 9/10 invariants and re-confirmed the invariant-4 breach; cycle-3 `deleg_7f56df3b` confirmed the ordering fix; cycle-4 `deleg_55f4e877` final re-review of all fixes. Fixes: engine fails closed without a PIT scorer; readiness blocks an explicit start before price history exists; each calendar day runs dividend_entitlement < dividend_payment < rebalance (shares bought on ex-date not entitled). Live probes: `GET /api/v1/backtest/readiness` on real data → ready=false, recommended_end=2026-08-27; `POST /api/v1/backtest/run` on real data (no vintages) → 400 + missing list; happy-path `scripts/probe_event_backtest.py` (synthetic PIT-complete) → accounting_reconciled=True, identity lhs=rhs, leakage_guard=True, 100-lot holdings.