kunthawat/set50-system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[verified] add event-study readiness gate

Browse Source
This commit is contained in:
Kunthawat Greethong
2026-08-23 12:02:22 +07:00
parent 91eb83631e
commit 0b47a06238
10 changed files with 396 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@@ -61,6 +61,7 @@ Data-health and replay endpoints:
GET /api/v1/data-health
GET /api/v1/vintages?as_of=<ISO-8601 timestamp>
GET /api/v1/replay/tourism?vintage_id=<vintage_id>
GET /api/v1/backtest/tourism?min_events=12
```
Source: `https://app.bot.or.th/BTWS_STAT/statistics/ReportPage.aspx?reportID=875&language=eng`
@@ -91,6 +92,8 @@ cd frontend && npm run build
- Raw response hash and normalized snapshot persistence
- Immutable vintage manifest with first-seen/revision metadata
- Read-only data-health, vintage timeline and vintage replay endpoints
- Deterministic event-study engine with benchmark and cost inputs
- Backtest readiness gate that blocks without independent vintages and prices
- Deterministic surprise × exposure × confidence score
- Paper ledger endpoint
- No LLM call yet; the deterministic result is the source of truth
@@ -98,3 +101,5 @@ cd frontend && npm run build
- No MT5 bridge yet
The next implementation step is the event-study/backtest layer using only vintages whose `published_at` is known at each test date.
The event-study gate is now exposed through `/api/v1/backtest/tourism`. It returns HTTP `409` with `status=blocked` when the independent-vintage minimum is not met, and it explicitly reports that a point-in-time daily price series is still required. The pure engine accepts events, daily prices, benchmark prices, event windows, and cost assumptions; it does not fetch or invent market prices.

19
backend/app/__init__.py
View File

@@ -14,11 +14,12 @@ from typing import Any
from flask import Flask, jsonify, request
from .bot_tourism import BotTourismSource, TourismSourceError
from .event_study import EventStudyError, assess_backtest_readiness
from .paper import PaperLedger
from .tourism import compute_tourism_signal
from .vintages import VintageStore, VintageStoreError
APP_VERSION = "0.3.0"
APP_VERSION = "0.4.0"
def _load_default_snapshot() -> dict[str, Any]:
@@ -175,6 +176,22 @@ def create_app(config: dict[str, Any] | None = None) -> Flask:
return jsonify({"error": str(exc)}), 400
return jsonify({"as_of": as_of, "count": len(entries), "vintages": entries})
@app.get("/api/v1/backtest/tourism")
def tourism_backtest_readiness():
raw_min_events = request.args.get("min_events", "12")
try:
min_events = int(raw_min_events)
readiness = assess_backtest_readiness(app.extensions["vintage_store"].list_vintages(), min_events)
except (ValueError, TypeError, EventStudyError) as exc:
return jsonify({"error": str(exc)}), 400
body = {
**readiness,
"theme": "tourism",
"price_series_required": True,
"next_action": "collect independent published vintages before running event study" if readiness["status"] == "blocked" else "provide point-in-time daily price series",
}
return jsonify(body), 409 if readiness["status"] == "blocked" else 200
@app.get("/api/v1/replay/tourism")
def replay_tourism():
vintage_id = request.args.get("vintage_id", "")

163
backend/app/event_study.py Normal file
View File

@@ -0,0 +1,163 @@
"""Deterministic event-study primitives with fail-closed readiness gates."""
from __future__ import annotations
import math
from datetime import date, datetime, timezone
from statistics import fmean
from typing import Any, Mapping, Sequence
class EventStudyError(ValueError):
"""Raised when an event study cannot be computed safely."""
def _parse_date(value: str) -> date:
try:
if "T" in value:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.date()
return date.fromisoformat(value)
except (TypeError, ValueError) as exc:
raise EventStudyError("date must be ISO-8601") from exc
def assess_backtest_readiness(vintages: Sequence[Mapping[str, Any]], min_events: int = 12) -> dict[str, Any]:
if min_events < 1:
raise EventStudyError("min_events must be positive")
ids = {str(item.get("vintage_id") or item.get("event_id") or "") for item in vintages}
ids.discard("")
available = len(ids)
if available < min_events:
return {
"status": "blocked",
"reason": "insufficient_vintages",
"available_events": available,
"required_events": min_events,
}
return {
"status": "ready",
"reason": None,
"available_events": available,
"required_events": min_events,
}
def _price_map(symbol: str, rows: Sequence[Mapping[str, Any]]) -> dict[date, float]:
if not rows:
raise EventStudyError(f"missing prices for {symbol}")
values: dict[date, float] = {}
for row in rows:
try:
trading_day = _parse_date(str(row["date"]))
close = float(row["close"])
except (KeyError, TypeError, ValueError) as exc:
raise EventStudyError(f"invalid price row for {symbol}") from exc
if not math.isfinite(close) or close <= 0:
raise EventStudyError(f"invalid close for {symbol}")
if trading_day in values:
raise EventStudyError(f"duplicate price date for {symbol}")
values[trading_day] = close
return dict(sorted(values.items()))
def _window_return(series: dict[date, float], event_date: date, window: int, symbol: str) -> float:
dates = list(series)
anchor_candidates = [index for index, trading_day in enumerate(dates) if trading_day >= event_date]
if not anchor_candidates:
raise EventStudyError(f"missing post-event prices for {symbol}")
anchor = anchor_candidates[0]
end = anchor + window
if end >= len(dates):
raise EventStudyError(f"insufficient price history for {symbol} window {window}")
return series[dates[end]] / series[dates[anchor]] - 1.0
def run_event_study(
events: Sequence[Mapping[str, Any]],
prices: Mapping[str, Sequence[Mapping[str, Any]]],
*,
benchmark_prices: Sequence[Mapping[str, Any]] | None = None,
windows: Sequence[int] = (1, 3, 5, 20),
cost_bps: float = 0.0,
min_events: int = 12,
) -> dict[str, Any]:
"""Calculate weighted post-publication returns from point-in-time events."""
readiness = assess_backtest_readiness(events, min_events=min_events)
if readiness["status"] != "ready":
raise EventStudyError(
f"insufficient vintages: {readiness['available_events']}/{readiness['required_events']}"
)
if not windows or any(isinstance(window, bool) or int(window) != window or window <= 0 for window in windows):
raise EventStudyError("windows must contain positive integers")
try:
cost_bps = float(cost_bps)
except (TypeError, ValueError) as exc:
raise EventStudyError("cost_bps must be finite and non-negative") from exc
if not math.isfinite(cost_bps) or cost_bps < 0:
raise EventStudyError("cost_bps must be finite and non-negative")
normalized_prices = {str(symbol).upper(): _price_map(str(symbol).upper(), rows) for symbol, rows in prices.items()}
normalized_benchmark = _price_map("benchmark", benchmark_prices) if benchmark_prices is not None else None
seen_event_ids: set[str] = set()
event_rows: list[dict[str, Any]] = []
for event in events:
event_id = str(event.get("event_id", ""))
if not event_id or event_id in seen_event_ids:
raise EventStudyError("event_id must be unique and non-empty")
seen_event_ids.add(event_id)
event_date = _parse_date(str(event.get("published_at", "")))
signals = event.get("signals")
if not isinstance(signals, Sequence) or isinstance(signals, (str, bytes)) or not signals:
raise EventStudyError(f"event {event_id} has no signals")
event_rows.append({"event_id": event_id, "event_date": event_date, "signals": signals})
window_results: dict[str, Any] = {}
for window in windows:
gross_returns: list[float] = []
net_returns: list[float] = []
benchmark_returns: list[float] = []
for event in event_rows:
gross = 0.0
turnover = 0.0
for signal in event["signals"]:
symbol = str(signal.get("symbol", "")).upper()
try:
weight = float(signal["target_weight"])
except (KeyError, TypeError, ValueError) as exc:
raise EventStudyError(f"invalid target weight for {symbol}") from exc
if not symbol or not math.isfinite(weight):
raise EventStudyError(f"invalid target weight for {symbol}")
if symbol not in normalized_prices:
raise EventStudyError(f"missing prices for {symbol}")
gross += weight * _window_return(normalized_prices[symbol], event["event_date"], int(window), symbol)
turnover += abs(weight)
cost = turnover * cost_bps / 10000.0
gross_returns.append(gross)
net_returns.append(gross - cost)
if normalized_benchmark is not None:
benchmark_returns.append(_window_return(normalized_benchmark, event["event_date"], int(window), "benchmark"))
average_gross = fmean(gross_returns)
average_net = fmean(net_returns)
average_benchmark = fmean(benchmark_returns) if benchmark_returns else None
window_results[str(window)] = {
"window_sessions": int(window),
"event_count": len(event_rows),
"gross_return": round(average_gross, 8),
"net_return": round(average_net, 8),
"benchmark_return": round(average_benchmark, 8) if average_benchmark is not None else None,
"active_return": round(average_net - average_benchmark, 8) if average_benchmark is not None else None,
"hit_rate": round(sum(value > 0 for value in net_returns) / len(net_returns), 8),
"cost_bps": cost_bps,
"event_returns": [round(value, 8) for value in net_returns],
}
return {
"status": "ready",
"event_count": len(event_rows),
"windows": window_results,
"min_events": min_events,
}

16
backend/tests/test_api.py
View File

@@ -95,6 +95,22 @@ class ApiTests(unittest.TestCase):
self.assertEqual(body["source_id"], "bot.ec_ei_028_s2")
self.assertEqual(body["status"], "provisional")
def test_backtest_readiness_blocks_without_enough_vintages(self):
with tempfile.TemporaryDirectory() as temp_dir:
store = VintageStore(Path(temp_dir))
store.persist(b"fixture raw", self.snapshot)
app = create_app({"TESTING": True, "SNAPSHOT": self.snapshot, "VINTAGE_STORE": store})
response = app.test_client().get("/api/v1/backtest/tourism?min_events=12")
body = response.get_json()
self.assertEqual(response.status_code, 409)
self.assertEqual(body["status"], "blocked")
self.assertEqual(body["reason"], "insufficient_vintages")
self.assertEqual(body["available_events"], 1)
def test_backtest_readiness_rejects_invalid_min_events(self):
response = self.client.get("/api/v1/backtest/tourism?min_events=bad")
self.assertEqual(response.status_code, 400)
def test_vintages_endpoint_filters_by_publication_timestamp(self):
with tempfile.TemporaryDirectory() as temp_dir:
store = VintageStore(Path(temp_dir))

52
backend/tests/test_event_study.py Normal file
View File

@@ -0,0 +1,52 @@
import unittest
from app.event_study import EventStudyError, assess_backtest_readiness, run_event_study
class EventStudyTests(unittest.TestCase):
def test_readiness_blocks_when_independent_events_are_insufficient(self):
result = assess_backtest_readiness([{"vintage_id": "v1"}], min_events=12)
self.assertEqual(result["status"], "blocked")
self.assertEqual(result["reason"], "insufficient_vintages")
self.assertEqual(result["available_events"], 1)
self.assertEqual(result["required_events"], 12)
def test_event_study_applies_windowed_returns_and_costs(self):
events = [
{
"event_id": "v1",
"published_at": "2026-01-01T08:00:00+07:00",
"signals": [
{"symbol": "AOT", "target_weight": 0.5},
{"symbol": "PTT", "target_weight": -0.5},
],
},
{
"event_id": "v2",
"published_at": "2026-01-01T08:00:00+07:00",
"signals": [
{"symbol": "AOT", "target_weight": 0.5},
{"symbol": "PTT", "target_weight": -0.5},
],
},
]
prices = {
"AOT": [{"date": "2026-01-01", "close": 100}, {"date": "2026-01-02", "close": 102}],
"PTT": [{"date": "2026-01-01", "close": 100}, {"date": "2026-01-02", "close": 99}],
}
result = run_event_study(events, prices, benchmark_prices=[{"date": "2026-01-01", "close": 100}, {"date": "2026-01-02", "close": 101}], windows=(1,), cost_bps=100, min_events=2)
window = result["windows"]["1"]
self.assertEqual(result["status"], "ready")
self.assertEqual(result["event_count"], 2)
self.assertGreater(window["gross_return"], window["net_return"])
self.assertEqual(window["cost_bps"], 100)
self.assertIsNotNone(window["benchmark_return"])
def test_event_study_rejects_missing_price_series(self):
events = [{"event_id": "v1", "published_at": "2026-01-01T08:00:00+07:00", "signals": [{"symbol": "AOT", "target_weight": 1.0}]}]
with self.assertRaisesRegex(EventStudyError, "missing prices"):
run_event_study(events, {}, windows=(1,), min_events=1)
if __name__ == "__main__":
unittest.main()

14
docs/HANDOFF.md
View File

@@ -5,10 +5,10 @@
- Path: `/Users/kunthawat/Gitea/set50-alternative-data-platform`
- Branch: `main`
- Verified code commit: `d1ba6ef` — `[verified] add vintage collector and point-in-time API`
- Current milestone: M2 vintage foundation complete; event study deferred
- Current milestone: M2.3 event-study foundation complete; real backtest blocked
- Mode: research + paper only
- Frontend: Vue 3 + Vite
- Backend: Flask `0.3.0`
- Backend: Flask `0.4.0`
- Current runtime source: BOT Tourism Indicators (`TOURISM_SOURCE=bot`)
## Completed
@@ -22,6 +22,8 @@
- Immutable `VintageStore` manifest with first-seen, last-seen, seen-count and revision metadata.
- One-shot collector: `backend/scripts/collect_tourism_vintage.py`.
- Point-in-time vintage query: `GET /api/v1/vintages?as_of=<ISO-8601>`.
- Deterministic event-study engine with window, benchmark and cost calculations.
- Backtest readiness gate: `GET /api/v1/backtest/tourism?min_events=12`.
- Ranked target weights and LONG/SHORT/NEUTRAL classification.
- English dashboard with live/provisional source label, sign-aware surprise copy and lineage fields.
- HttpOnly paper session and internal paper ledger.
@@ -53,7 +55,7 @@ npm run build
Vite build completed successfully.
GET /api/v1/health
HTTP 200; {"mode":"research","status":"ok","version":"0.3.0"}
HTTP 200; {"mode":"research","status":"ok","version":"0.4.0"}
GET /api/v1/data-health
HTTP 200; source_mode=bot, status=provisional, replayable=true
@@ -66,6 +68,9 @@ HTTP 200; count=0
GET /api/v1/vintages?as_of=2026-08-01T00:00:00Z
HTTP 200; count=1; manifest seen_count=4
GET /api/v1/backtest/tourism?min_events=12
HTTP 409; status=blocked, available_events=1, required_events=12, price_series_required=true
```
Paper writes use a server-side token exchange and HttpOnly `paper_session` cookie; the token is not embedded in the frontend bundle.
@@ -77,8 +82,9 @@ Paper writes use a server-side token exchange and HttpOnly `paper_session` cooki
- Snapshot storage is local filesystem and single-process; shared persistence is required before multi-worker deployment.
- No investment edge, transaction-cost result, or backtest conclusion has been established.
- One independent source release is not enough for a valid event study; current historical rows are not treated as point-in-time vintages.
- The event-study engine is deterministic and tested, but no real price provider is connected yet.
- Browser screenshot verification remains blocked by the Chrome remote-debugging permission prompt; served HTML/source, live API, fresh Vite build and replay integrity were verified instead.
## Exact next action
Collect independent BOT releases over time, then implement event-study/backtest checks using only vintages whose `published_at` is known at each test date. Add another metric only when its historical release coverage is real.
Collect independent BOT releases over time and add a point-in-time daily price provider. Only then raise the readiness gate and run the event study. Add another metric only when its historical release coverage is real.

5
docs/engineering-log.md
View File

@@ -7,6 +7,7 @@
| M0 repo foundation | complete | Flask API, Vue/Vite shell | keep research/paper guardrails |
| M1 BOT Tourism adapter | complete | 18 tests, live BOT fetch, raw/snapshot persistence | validate multiple vintages |
| M2 vintage collector | complete | 25 tests, manifest idempotency, live collector and point-in-time API | collect independent releases |
| M2.3 event-study gate | complete/blocked | 30 tests, pure engine and truthful 409 readiness API | add point-in-time price provider |
| Tourism deterministic signal | complete | live foreign-arrivals YoY surprise | add occupancy/airport metric |
| Internal paper ledger | complete | POST/readback through live API | persist in PostgreSQL later |
| Dashboard | complete | Vite build + served source check with live-sign copy | visual browser capture after permission is available |
@@ -24,7 +25,7 @@
## Verification
- Backend: 25 unittest tests pass.
- Backend: 30 unittest tests pass.
- Independent M1 review: **PASSED**; no concrete security or logic blockers.
- Reviewer suggestions: set `PAPER_COOKIE_SECURE=1` outside local HTTP; replace in-memory sessions before multi-worker deployment.
- M1 reviewer backlog: add schema-drift, duplicate/reordered-row, and malformed-vintage regression fixtures.
@@ -37,4 +38,6 @@
- Live vintage replay returned the same theme surprise as the current dashboard summary.
- Vintage collector preserved one live `vintage_id` with `seen_count=4` and point-in-time API excluded it before `published_at`.
- Independent M2 review: **PASSED**; no concrete security or logic blockers.
- Event-study readiness gate correctly returns HTTP 409 with 1/12 independent vintages; no backtest result is fabricated.
- Independent M2.3 review: **PASSED**; no concrete security or logic blockers.
- Browser visual capture was blocked by Chrome remote-debugging permission; no permission dialog was clicked.

68
docs/engineering-log/2026-08-23-event-study-gate.md Normal file
View File

@@ -0,0 +1,68 @@
# 2026-08-23 — event-study engine and backtest readiness gate
## Plan status
- Deterministic event-study engine: complete.
- Cost-aware window calculations: complete.
- Point-in-time vintage sufficiency gate: complete.
- Market-price provider and real backtest: blocked by design until independent releases and a point-in-time daily price series exist.
## Changed files
- `backend/app/event_study.py` — readiness gate, event-window returns, weighted portfolio returns, benchmark comparison, turnover cost and hit-rate calculations.
- `backend/app/__init__.py` — `/api/v1/backtest/tourism` readiness endpoint and API version `0.4.0`.
- `backend/tests/test_event_study.py` — sufficiency, window/cost and missing-price tests.
- `backend/tests/test_api.py` — blocked and invalid-query endpoint tests.
- `frontend/src/App.vue` — Backtest gate KPI that accepts HTTP 409 as a truthful blocked state instead of treating it as a dashboard failure.
- `README.md` — engine contract and readiness behavior.
## Contract
The engine accepts point-in-time events, daily symbol prices, an optional benchmark, event windows, and cost assumptions. It does not fetch prices, infer missing closes, or convert current revised historical data into past knowledge.
When fewer than the configured minimum events exist, the API returns:
```json
{
"status": "blocked",
"reason": "insufficient_vintages",
"available_events": 1,
"required_events": 12,
"price_series_required": true
}
```
## Live evidence
```text
GET /api/v1/backtest/tourism?min_events=12
HTTP 409
status: blocked
available_events: 1
required_events: 12
next_action: collect independent published vintages before running event study
```
The dashboard served source includes `fetchBacktestReadiness`, `Backtest gate`, and the blocked-vintage count. HTTP 409 is handled as an expected research state, not a page error.
## Verification
- `PYTHONPATH=backend .venv/bin/python -W error -m unittest discover -s backend/tests -v` — **30 tests passed**.
- `npm run build` — passed.
- Live API readiness gate — passed with truthful 409 blocked result.
- Invalid `min_events` — HTTP 400.
- Pure event-study fixture — window return, benchmark and cost calculation passed.
## Independent review
```text
passed: true
security_concerns: []
logic_errors: []
```
Non-blocking backlog: add API boundary cases for zero/negative `min_events` and multi-vintage benchmark/window gaps.
## Risks and exact next action
There is only one independent BOT release. No investment backtest result exists. Continue collecting releases and add a point-in-time daily price adapter before raising the readiness threshold or interpreting event-study output.

42
docs/test-evidence/2026-08-23-event-study-gate.md Normal file
View File

@@ -0,0 +1,42 @@
# Test evidence — 2026-08-23 event-study gate
## Automated
```text
PYTHONPATH=backend .venv/bin/python -W error -m unittest discover -s backend/tests -v
Ran 30 tests ... OK
npm run build
Vite build completed successfully.
```
## Live readiness
```text
GET /api/v1/backtest/tourism?min_events=12
HTTP 409
status=blocked
available_events=1
required_events=12
price_series_required=true
```
The 409 is intentional: the API refuses to produce an event study with one independent vintage.
## Engine fixture
The pure event-study fixture verified one-session window returns, weighted long/short portfolio aggregation, benchmark comparison, turnover cost deduction, hit rate, and missing-price rejection.
## Frontend served verification
The frontend build and served `/src/App.vue` contained the readiness loader and `Backtest gate` KPI. Browser screenshot capture remains unavailable because the Chrome remote-debugging permission prompt has not been approved.
## Independent review
```text
passed: true
security_concerns: []
logic_errors: []
```
Non-blocking backlog: add API boundary cases for zero/negative `min_events` and multi-vintage benchmark/window gaps.

19
frontend/src/App.vue
View File

@@ -5,6 +5,7 @@ const summary = ref(null)
const observations = ref(null)
const signalData = ref(null)
const ledger = ref({ entries: [] })
const backtest = ref(null)
const loading = ref(true)
const error = ref('')
const notice = ref('')
@@ -47,22 +48,33 @@ async function fetchJson(url, options) {
return response.json()
}
async function fetchBacktestReadiness() {
const response = await fetch('/api/v1/backtest/tourism?min_events=12')
const body = await response.json().catch(() => ({}))
if (![200, 409].includes(response.status)) {
throw new Error(body.error || `Request failed: ${response.status}`)
}
return body
}
async function loadDashboard() {
loading.value = true
error.value = ''
try {
const [summaryBody, observationBody, signalBody, ledgerBody, sessionBody] = await Promise.all([
const [summaryBody, observationBody, signalBody, ledgerBody, sessionBody, backtestBody] = await Promise.all([
fetchJson('/api/v1/dashboard/summary'),
fetchJson('/api/v1/factors/tourism/observations'),
fetchJson('/api/v1/signals'),
fetchJson('/api/v1/paper/ledger'),
fetchJson('/api/v1/auth/paper', { credentials: 'include' }),
fetchBacktestReadiness(),
])
summary.value = summaryBody
observations.value = observationBody
signalData.value = signalBody
ledger.value = ledgerBody
paperAuthenticated.value = Boolean(sessionBody.authenticated)
backtest.value = backtestBody
} catch (caught) {
error.value = caught.message
} finally {
@@ -205,6 +217,11 @@ onMounted(loadDashboard)
<div class="kpi-value quality-value">{{ summary.data_health.status }}</div>
<div class="kpi-foot">Vintage {{ summary.data_health.vintage_id }}</div>
</article>
<article class="kpi-card">
<div class="kpi-label">Backtest gate</div>
<div class="kpi-value quality-value">{{ backtest?.status || '—' }}</div>
<div class="kpi-foot">{{ backtest?.available_events || 0 }} / {{ backtest?.required_events || 0 }} vintages</div>
</article>
</section>
<section class="hero-grid" id="factors">