[verified] Task 8 close: fix price-coverage-at-start + engine fail-closed (review cycle-1) + docs (54 tests, cycle-4 passed)

backend/app/backtest_engine.py

@@ -181,11 +181,25 @@ def run_event_backtest(
     # Merge all dated actions into one timeline.
     # Each day holds a list of (action, payload) where action is one of
     # "rebalance", "dividend_entitlement", "dividend_payment".
+    #
+    # Within a day, actions must run in a market-correct order that honours the
+    # invariants:
+    #   1) dividend_entitlement BEFORE rebalance — an entitlement is captured
+    #      from the position held *before* that day's trades, so shares acquired
+    #      on the ex-date do NOT qualify (confirmed invariant #4);
+    #   2) dividend_payment BEFORE rebalance — cash that settles on this day is
+    #      available to fund that day's buys;
+    #   3) rebalance last.
+    # We sort each day's actions by this priority rather than trusting insertion
+    # order.
+    _ORDER = {"dividend_entitlement": 0, "dividend_payment": 1, "rebalance": 2}
     timeline: dict[dt.date, list[tuple[str, Any]]] = {}
-    for exec_day, sig in exec_map.items():
-        timeline.setdefault(exec_day, []).append(("rebalance", sig))
     for ev in dividend_events:
         timeline.setdefault(ev.date, []).append((ev.kind, ev))
+    for exec_day, sig in exec_map.items():
+        timeline.setdefault(exec_day, []).append(("rebalance", sig))
+    for day in list(timeline.keys()):
+        timeline[day].sort(key=lambda pair: _ORDER.get(pair[0], 3))
 
     last_scores: dict = {}
     leakage_guard = False

backend/app/backtest_readiness.py

@@ -274,6 +274,11 @@ def evaluate_readiness(
     if start is not None:
         s = _parse_date(start)
         e_ = _parse_date(end) if end else today
+        # price must be executable AT the start: an explicit start that predates
+        # all usable price history would otherwise manufacture a false PIT
+        # window (factor + Siamchart are already checked at start above).
+        if pc.available and pc.earliest is not None and s < pc.earliest:
+            miss.append("price")
         ready = (not miss) and s <= e_
     else:
         ready = (not miss) and bool(recommended_start)

backend/tests/test_backtest_engine.py

@@ -171,6 +171,43 @@ class ResultContractTest(unittest.TestCase):
                 symbols=["A"],  # no score_fn -> must raise
             )
 
+    def test_no_entitlement_for_shares_bought_on_ex_date(self):
+        # Regression for the cycle-2 review breach (invariant #4): when an
+        # execution day coincides with a dividend ex-date, shares acquired
+        # ON the ex-date must not be entitled to that dividend. The engine must
+        # capture entitlement from the pre-trade position.
+        # Daily price series; A held from day 1. ex-date on a day when an
+        # initial signal's next trading day also lands (coincident) — the buy on
+        # that ex-date must NOT count toward entitlement.
+        ledger = FakeLedger([{
+            "symbol": "A", "ex_date": "2026-01-02",
+            "per_share": 1.0, "estimate": False,
+        }])
+        # start next trading day after 2026-01-01 is 2026-01-02 (daily series),
+        # which is the exact ex-date -> coincident execution + ex-date.
+        series = make_series(["A", "B"], "2026-01-01", 30)
+        res = run_event_backtest(
+            start="2026-01-01", end="2026-03-31",
+            capital=200_000, dividend_ledger=ledger,
+            price_series=series, score_fn=noop_scorer, symbols=["A", "B"],
+        )
+        # dividend was entitled for A, but only for shares held BEFORE the
+        # ex-date. Since A was FIRST bought on the ex-date (initial signal's
+        # next trading day), entitlement must be 0 (no receivable created).
+        assert res.ledger is not None
+        # no receivable for A on the ex-date (shares only acquired that day)
+        a_receivables = [
+            r for r in res.ledger.state.receivables
+            if r.symbol == "A" and r.ex_date == "2026-01-02"
+        ]
+        self.assertEqual(a_receivables, [], "shares bought on ex-date must not be entitled")
+        # no dividend log entry either
+        for d in (res.ledger.state.dividends or []):
+            if d["symbol"] == "A" and d["ex_date"] == "2026-01-02":
+                self.fail("dividend logged for shares bought on ex-date")
+        # reconcile still holds
+        self.assertTrue(res.account_reconciled)
+
 
 if __name__ == "__main__":
     unittest.main()

backend/tests/test_backtest_readiness.py

@@ -161,6 +161,21 @@ class ExplicitWindowTest(unittest.TestCase):
         self.assertFalse(res.ready)
         self.assertIn("factor", res.missing)
 
+    def test_explicit_start_before_price_history_blocks(self):
+        # Regression (cycle-1 review): explicit start that predates ALL usable
+        # price history must not manufacture a false strict-PIT window. Factor +
+        # Siamchart are ready at the start, but price is not -> block.
+        store = full_factor_store("2024-01-01")
+        sc = FakeSiamchartStore(["2024-01-01T09:00:00+07:00"])
+        # price only starts 2026-06-01
+        series = make_price_series(["A"], "2026-06-01", "2026-08-01")
+        res = evaluate_readiness(
+            factor_store=store, siamchart_store=sc, price_series=series,
+            start="2025-01-01", end="2026-08-01",
+        )
+        self.assertFalse(res.ready)
+        self.assertIn("price", res.missing)
+
     def test_explicit_start_after_readiness_is_ready(self):
         store = full_factor_store("2025-01-01")
         sc = FakeSiamchartStore(["2025-01-01T09:00:00+07:00"])

docs/HANDOFF.md

@@ -180,12 +180,12 @@ engine per the user's description. Confirmed decisions:
 - `d73a58b` Task 7 frontend — readiness gate disables Run + shows missing coverage; detailed report.
 
 ### Verification
-- Standalone offline suite **52 tests** (readiness 9, events 14, ledger 15, rebalancer 5, engine 5, store 4). compileall + diff-check clean.
+- Standalone offline suite **54 tests** (readiness 10, events 14, ledger 15, rebalancer 5, engine 6, store 4). compileall + diff-check clean.
 - Static scan: no secrets/eval/debug (4 "secret" regex hits are the prose "missing token" string).
 - Live readiness on real data: `ready=false`, missing factor+siamchart, recommended_end=2026-08-27.
 - Live strict run on real data (no vintages): `POST /api/v1/backtest/run` → **400** + missing list (fail-closed).
 - Happy-path `scripts/probe_event_backtest.py` (synthetic PIT-complete): accounting_reconciled=True, identity lhs=rhs, leakage_guard=True, 100-lot holdings.
-- Cycle-1 independent review `deleg_5225e9de` flagged: engine's no-scorer fallback used the live board (look-ahead). Fixed to fail-closed. Cycle-2 review `deleg_b969075c` re-verifying.
+- Independent reviews: cycle-1 `deleg_5225e9de` flagged 3 logic errors → fixed (engine fails closed without PIT scorer; readiness blocks explicit start before price history; coincident-day ex-date entitlement fixed by per-day ordering). Cycle-2 `deleg_b969075c` re-verified the fallback fix + 9/10 invariants, re-confirmed the invariant-4 breach. Cycle-3 `deleg_7f56df3b` confirmed the ordering fix. Cycle-4 `deleg_55f4e877` final re-review of all three fixes + all invariants.
 - Evidence: `docs/test-evidence/2026-08-28-event-driven-backtest.md`.
 
 ### Honest scope / limitations

docs/engineering-log.md

@@ -106,4 +106,4 @@
 - Real dated dividend-history collector (2026-08-27, commit `f9973e8`): `siamchart.py` gained `parse_dividend_history`/`fetch_dividend_history` (reads the "ประวัติการปันผล" ex-date + DPS table per stock-info page); `dividend_ledger.py` gained `populate_dated_dividends` (registers dated rows, estimate=False). `POST /api/v1/dividends/update` fetches all snapshot symbols and persists a dated ledger at `data/dividends/ledger.json`; `use_ledger` backtests prefer the dated ledger (`dividend_method=dated_ledger`) and fall back to DPS estimates when unpopulated. Full backend **292 passed** (was 286). Live network fetch: 49/49 symbols, 1410 dated payments; use_ledger → `dated_ledger`. Note: this sandbox HAS outbound network (curl/https to siamchart 200) — contrary to earlier assumption.
 - Auto-refresh dated dividend ledger (2026-08-27, commit `03195dc`): `AppDataScheduler` now calls `_maybe_refresh_dated_dividends()` in `refresh_all` — a cooldown-gated (default 6h) fetch of real dated dividend history into `data/dividends/ledger.json`, so the ledger stays fresh without manual `/api/v1/dividends/update` and without hammering the source every tick. `DIVIDEND_REFRESH_COOLDOWN_SECONDS` env toggle. Full backend **294 passed** (was 292).
 - Single-container Docker packaging (2026-08-27, commit `f9b8cd1`): added `Dockerfile` (python:3.11-slim + nginx; serves the PREBUILT `frontend/dist` SPA, proxies `/api` → 127.0.0.1:5000, `VOLUME /app/backend/data`, HEALTHCHECK on /api/v1/dashboard/summary, exposes :80), `deploy/nginx.conf`, `deploy/entrypoint.sh` (Flask HOST=0.0.0.0:5000 + nginx foreground), `docker-compose.yml` (port 8080:80, volume `./backend/data`), `.dockerignore`, and `.gitignore` now tracks `frontend/dist/` (prebuilt bundle required by the image; `backend/data/` stays untracked). Backend runtime verified (test_client `/api/v1/dashboard/summary` = 200). No local docker on this machine so the image itself must build on EasyPanel.
-- Event-driven PIT backtest (2026-08-28, commits `68f2cc1` → `d73a58b`, 8-phase plan at `.hermes/plans/2026-08-28_091000-event-driven-pit-backtest.md`): replaces the calendar-rebalance backtest with a strict point-in-time event-driven engine. Confirmed user decisions: (1) strict PIT — block when factor/Siamchart/price coverage is incomplete; (2) execute at next trading-day close after release (no same-day look-ahead); (3) dividend cash available ex_date+30 calendar days (`ex_date_plus_30d` assumption, not an observed pay date — Siamchart source gives ex-date+DPS only); (4) average-cost realized P&L; (5) all-in 0.3% fee per trade, no added tax. Ships: `backtest_readiness.py` (readiness + recommended start/end defaults, fail-closed), `backtest_events.py` (event calendar + next-trading-day mapping), `portfolio_ledger.py` (cash/positions/avg-cost/realized+unrealized P&L/dividend receivables/fees; gross realized + fees subtracted once), `portfolio_rebalancer.py` (50/20/30 → executable 100-lot orders, sells-first, no trade on unchanged target), `backtest_engine.py` (event processor; refuses to fall back to the live board without a PIT scorer — fail-closed, was flagged by cycle-1 review `deleg_5225e9de` and fixed), `backtest_store.py` (durable atomic JSON runs at `data/backtest/runs.json`), strict route `POST /api/v1/backtest/run` (readiness-gate → 400 with missing list when not ready) + `GET /api/v1/backtest/run(+<id>)`, and frontend (readiness gate disables Run + shows missing coverage; report shows realized/unrealized P&L, dividend received/receivable, fees, final holdings with avg cost/last price/unrealized). Offline standalone suite **52 tests** (9+14+15+5+5+4); compileall + diff-check clean; static scan no secrets/eval/debug (4 "secret"-regex hits are the prose "missing token" string, not credentials). Live probes: `GET /api/v1/backtest/readiness` on real data → ready=false, recommended_end=2026-08-27; `POST /api/v1/backtest/run` on real data (no vintages) → 400 + missing list; happy-path `scripts/probe_event_backtest.py` (synthetic PIT-complete) → accounting_reconciled=True, identity lhs=rhs, leakage_guard=True, 100-lot holdings.
+- Event-driven PIT backtest (2026-08-28, commits `68f2cc1` → `d73a58b`, 8-phase plan at `.hermes/plans/2026-08-28_091000-event-driven-pit-backtest.md`): replaces the calendar-rebalance backtest with a strict point-in-time event-driven engine. Confirmed user decisions: (1) strict PIT — block when factor/Siamchart/price coverage is incomplete; (2) execute at next trading-day close after release (no same-day look-ahead); (3) dividend cash available ex_date+30 calendar days (`ex_date_plus_30d` assumption, not an observed pay date — Siamchart source gives ex-date+DPS only); (4) average-cost realized P&L; (5) all-in 0.3% fee per trade, no added tax. Ships: `backtest_readiness.py` (readiness + recommended start/end defaults, fail-closed), `backtest_events.py` (event calendar + next-trading-day mapping), `portfolio_ledger.py` (cash/positions/avg-cost/realized+unrealized P&L/dividend receivables/fees; gross realized + fees subtracted once), `portfolio_rebalancer.py` (50/20/30 → executable 100-lot orders, sells-first, no trade on unchanged target), `backtest_engine.py` (event processor; refuses to fall back to the live board without a PIT scorer — fail-closed, was flagged by cycle-1 review `deleg_5225e9de` and fixed), `backtest_store.py` (durable atomic JSON runs at `data/backtest/runs.json`), strict route `POST /api/v1/backtest/run` (readiness-gate → 400 with missing list when not ready) + `GET /api/v1/backtest/run(+<id>)`, and frontend (readiness gate disables Run + shows missing coverage; report shows realized/unrealized P&L, dividend received/receivable, fees, final holdings with avg cost/last price/unrealized). Offline standalone suite **54 tests** (9+14+15+5+6+4→ readiness 10 after price regression); compileall + diff-check clean; static scan no secrets/eval/debug (4 "secret"-regex hits are the prose "missing token" string, not credentials). Independent review cycles: cycle-1 `deleg_5225e9de` flagged 3 logic errors (no-scorer live-board fallback look-ahead; price_coverage not checking explicit start → false PIT window; coincident-day ex-date entitlement breach) → all fixed; cycle-2 `deleg_b969075c` re-verified the fallback fix + 9/10 invariants and re-confirmed the invariant-4 breach; cycle-3 `deleg_7f56df3b` confirmed the ordering fix; cycle-4 `deleg_55f4e877` final re-review of all fixes. Fixes: engine fails closed without a PIT scorer; readiness blocks an explicit start before price history exists; each calendar day runs dividend_entitlement < dividend_payment < rebalance (shares bought on ex-date not entitled). Live probes: `GET /api/v1/backtest/readiness` on real data → ready=false, recommended_end=2026-08-27; `POST /api/v1/backtest/run` on real data (no vintages) → 400 + missing list; happy-path `scripts/probe_event_backtest.py` (synthetic PIT-complete) → accounting_reconciled=True, identity lhs=rhs, leakage_guard=True, 100-lot holdings.