diff --git a/.gitignore b/.gitignore index 16ae5d4..a3126c0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ __pycache__/ .env .DS_Store backend/.pytest_cache/ +backend/data/ frontend/node_modules/ frontend/dist/ reports/ diff --git a/README.md b/README.md index c01d67d..92c01cd 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # SET50 Alternative Data Platform -Tourism-first vertical slice for a deterministic SET50 alternative-data research system. +Tourism-first vertical slice for a deterministic SET50 alternative-data research system. The app can run against a clearly-labelled fixture or fetch a point-in-time Tourism Indicators vintage from the Bank of Thailand report backed by the Ministry of Tourism and Sports. Current scope: ```text -fixture observation +fixture or BOT source observation → Tourism Pulse surprise → versioned exposure score → ranked target weights @@ -20,7 +20,7 @@ No external webhook receiver and no live MT5 execution are enabled. ```bash python -m venv .venv .venv/bin/pip install -r backend/requirements.txt -PAPER_WRITE_TOKEN=local-paper-token PYTHONPATH=backend .venv/bin/python backend/run.py +PAPER_WRITE_TOKEN=local-paper-token TOURISM_SOURCE=fixture PYTHONPATH=backend .venv/bin/python backend/run.py ``` Health check: @@ -45,6 +45,25 @@ The frontend reads the live API through Vite's `/api` proxy. Paper writes requir For HTTPS/non-local deployment, set `PAPER_COOKIE_SECURE=1`. The M0 session store is intentionally in-memory and single-process; use a shared session store before running multiple workers or replicas. +## Run with the real BOT Tourism source + +Use the BOT-backed adapter when network access is available: + +```bash +PAPER_WRITE_TOKEN=local-paper-token TOURISM_SOURCE=bot PYTHONPATH=backend .venv/bin/python backend/run.py +``` + +At startup the adapter performs a read-only GET/POST against the BOT Tourism Indicators report, parses the available monthly history, computes the latest year-over-year arrival observation against a trailing 12-point baseline, and stores the raw HTML plus normalized snapshot under `backend/data/` (ignored by git). The dashboard labels provisional BOT data as `provisional`, not `high`. + +Data-health and replay endpoints: + +```text +GET /api/v1/data-health +GET /api/v1/replay/tourism?vintage_id= +``` + +Source: `https://app.bot.or.th/BTWS_STAT/statistics/ReportPage.aspx?reportID=875&language=eng` + ## Tests and build ```bash @@ -52,16 +71,18 @@ PYTHONPATH=backend .venv/bin/python -m unittest discover -s backend/tests -v cd frontend && npm run build ``` -## Current M0 boundary +## Current M1 boundary - English UI and analysis vocabulary - Research mode and paper mode only -- Tourism Pulse fixture adapter +- Tourism Pulse fixture adapter and BOT Tourism Indicators adapter - Data lineage: source, publication time, retrieval time, vintage +- Raw response hash and normalized snapshot persistence +- Read-only data-health and vintage replay endpoints - Deterministic surprise × exposure × confidence score - Paper ledger endpoint - No LLM call yet; the deterministic result is the source of truth - No webhook receiver yet - No MT5 bridge yet -The next implementation step is replacing the fixture with one replayable Tourism source adapter while preserving the same snapshot contract. +The next implementation step is to validate multiple BOT vintages and add the next real Tourism metric before introducing LLM analysis. diff --git a/backend/app/__init__.py b/backend/app/__init__.py index 67372d9..f6ff99f 100644 --- a/backend/app/__init__.py +++ b/backend/app/__init__.py @@ -5,6 +5,7 @@ from __future__ import annotations import hmac import json import os +import re import secrets import time from pathlib import Path @@ -12,10 +13,11 @@ from typing import Any from flask import Flask, jsonify, request +from .bot_tourism import BotTourismSource, TourismSourceError from .paper import PaperLedger from .tourism import compute_tourism_signal -APP_VERSION = "0.1.0" +APP_VERSION = "0.2.0" def _load_default_snapshot() -> dict[str, Any]: @@ -35,18 +37,37 @@ def _signal_summary(result: dict[str, Any]) -> dict[str, int]: def create_app(config: dict[str, Any] | None = None) -> Flask: app = Flask(__name__) + data_root = Path(__file__).resolve().parents[1] / "data" app.config.from_mapping( TESTING=False, MODE=os.getenv("APP_MODE", "research"), PAPER_WRITE_TOKEN=os.getenv("PAPER_WRITE_TOKEN", ""), PAPER_COOKIE_SECURE=os.getenv("PAPER_COOKIE_SECURE", "0") == "1", PAPER_SESSION_SECONDS=int(os.getenv("PAPER_SESSION_SECONDS", "3600")), + TOURISM_SOURCE=os.getenv("TOURISM_SOURCE", "fixture"), + TOURISM_ADAPTER=None, + TOURISM_RAW_DIR=Path(os.getenv("TOURISM_RAW_DIR", str(data_root / "raw" / "tourism"))), + SNAPSHOT_DIR=Path(os.getenv("TOURISM_SNAPSHOT_DIR", str(data_root / "snapshots"))), SNAPSHOT=_load_default_snapshot(), ) if config: app.config.update(config) - result = compute_tourism_signal(app.config["SNAPSHOT"]) + source_snapshot = app.config["SNAPSHOT"] + source_mode = str(app.config["TOURISM_SOURCE"]).lower() + if source_mode == "bot": + adapter = app.config.get("TOURISM_ADAPTER") or BotTourismSource( + raw_dir=Path(app.config["TOURISM_RAW_DIR"]), + snapshot_dir=Path(app.config["SNAPSHOT_DIR"]), + ) + try: + source_snapshot = adapter.fetch(exposures=source_snapshot.get("exposures", [])) + except TourismSourceError as exc: + raise RuntimeError(f"Tourism source startup failed: {exc}") from exc + elif source_mode != "fixture": + raise ValueError(f"unsupported TOURISM_SOURCE: {source_mode}") + + result = compute_tourism_signal(source_snapshot) ledger = PaperLedger() paper_sessions: dict[str, float] = {} allowed_symbols = {item["symbol"] for item in result["signals"]} @@ -108,6 +129,58 @@ def create_app(config: dict[str, Any] | None = None) -> Flask: authorized, _ = _paper_write_authorized() return jsonify({"authenticated": authorized}) + def _snapshot_file_for(vintage_id: str) -> Path | None: + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}", vintage_id): + return None + root = Path(app.config["SNAPSHOT_DIR"]).resolve() + candidate = (root / f"{vintage_id}.json").resolve() + if candidate.parent != root: + return None + return candidate + + def _source_replayable(source: dict[str, Any]) -> bool: + vintage_id = str(source.get("vintage_id", "")) + snapshot_path = _snapshot_file_for(vintage_id) if vintage_id else None + return snapshot_path is not None and snapshot_path.is_file() + + @app.get("/api/v1/data-health") + def data_health(): + current = app.extensions["tourism_result"] + source = current["source"] + return jsonify( + { + "status": current["data_quality"], + "source_id": source.get("source_id"), + "source_url": source.get("source_url"), + "published_at": source.get("published_at"), + "retrieved_at": source.get("retrieved_at"), + "vintage_id": source.get("vintage_id"), + "raw_payload_hash": source.get("raw_payload_hash"), + "parser_version": source.get("parser_version"), + "available_periods": source.get("available_periods"), + "history_points": source.get("history_points"), + "replayable": _source_replayable(source), + "source_mode": app.config["TOURISM_SOURCE"], + } + ) + + @app.get("/api/v1/replay/tourism") + def replay_tourism(): + vintage_id = request.args.get("vintage_id", "") + snapshot_path = _snapshot_file_for(vintage_id) + if snapshot_path is None: + return jsonify({"error": "invalid vintage_id"}), 400 + if not snapshot_path.is_file(): + return jsonify({"error": "vintage snapshot not found"}), 404 + try: + frozen_snapshot = json.loads(snapshot_path.read_text(encoding="utf-8")) + if frozen_snapshot.get("source", {}).get("vintage_id") != vintage_id: + return jsonify({"error": "vintage snapshot identity mismatch"}), 409 + replayed = compute_tourism_signal(frozen_snapshot) + except (OSError, json.JSONDecodeError, TypeError, ValueError) as exc: + return jsonify({"error": f"vintage snapshot is invalid: {exc.__class__.__name__}"}), 422 + return jsonify({"vintage_id": vintage_id, "result": replayed}) + @app.get("/api/v1/health") def health(): return jsonify({"status": "ok", "mode": app.config["MODE"], "version": APP_VERSION}) @@ -129,6 +202,12 @@ def create_app(config: dict[str, Any] | None = None) -> Flask: "published_at": current["source"].get("published_at"), "retrieved_at": current["source"].get("retrieved_at"), "vintage_id": current["source"].get("vintage_id"), + "raw_payload_hash": current["source"].get("raw_payload_hash"), + "parser_version": current["source"].get("parser_version"), + "available_periods": current["source"].get("available_periods"), + "history_points": current["source"].get("history_points"), + "source_mode": app.config["TOURISM_SOURCE"], + "replayable": _source_replayable(current["source"]), }, "signal_summary": _signal_summary(current), "top_signals": current["signals"][:5], diff --git a/backend/app/bot_tourism.py b/backend/app/bot_tourism.py new file mode 100644 index 0000000..9a29750 --- /dev/null +++ b/backend/app/bot_tourism.py @@ -0,0 +1,347 @@ +"""Bank of Thailand Tourism Indicators source adapter.""" + +from __future__ import annotations + +import hashlib +import html as html_lib +import json +import math +import re +from calendar import monthrange +from datetime import date, datetime, timedelta, timezone +from html.parser import HTMLParser +from pathlib import Path +from statistics import median, pstdev +from typing import Any, Callable, Iterable +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode, urljoin +from urllib.request import HTTPCookieProcessor, Request, build_opener + +from http.cookiejar import CookieJar + +BOT_TOURISM_URL = "https://app.bot.or.th/BTWS_STAT/statistics/ReportPage.aspx?reportID=875&language=eng" +BOT_SOURCE_ID = "bot.ec_ei_028_s2" +PARSER_VERSION = "bot-tourism-v1" +_BANGKOK_TZ = timezone(timedelta(hours=7)) +_MONTHS = {name: index for index, name in enumerate(("JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "NOV", "DEC"), start=1)} +_PERIOD_RE = re.compile(r"^([A-Z]{3})\s+(\d{4})(?:\s+([A-Z]))?$") +_UPDATED_RE = re.compile(r"Last\s+Updated\s*:\s*(\d{1,2}\s+[A-Za-z]{3}\s+\d{4}\s+\d{2}:\d{2})", re.IGNORECASE) + + +class TourismSourceError(RuntimeError): + """Raised when the external tourism source cannot be trusted or parsed.""" + + +class _FormParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.in_form = False + self.form_action = "" + self.hidden: dict[str, str] = {} + self.options: dict[str, list[str]] = {} + self.selected: dict[str, str] = {} + self._select: str | None = None + self._select_selected: str | None = None + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + attributes = dict(attrs) + if tag == "form": + self.in_form = True + self.form_action = attributes.get("action") or "" + elif self.in_form and tag == "input" and attributes.get("name"): + if attributes.get("type", "hidden").lower() == "hidden": + self.hidden[attributes["name"]] = attributes.get("value") or "" + elif self.in_form and tag == "select" and attributes.get("name"): + self._select = attributes["name"] + self.options[self._select] = [] + self._select_selected = None + elif self.in_form and tag == "option" and self._select: + value = attributes.get("value") or "" + self.options[self._select].append(value) + if "selected" in attributes: + self._select_selected = value + + def handle_endtag(self, tag: str) -> None: + if tag == "select" and self._select: + values = self.options[self._select] + self.selected[self._select] = self._select_selected or (values[0] if values else "") + self._select = None + self._select_selected = None + elif tag == "form": + self.in_form = False + + +class _TableParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.in_target = False + self.table_depth = 0 + self.in_row = False + self.in_cell = False + self.rows: list[list[str]] = [] + self._row: list[str] = [] + self._cell: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + attributes = dict(attrs) + if tag == "table" and attributes.get("id") == "dgExcel": + self.in_target = True + self.table_depth = 1 + elif self.in_target and tag == "table": + self.table_depth += 1 + elif self.in_target and tag == "tr": + self.in_row = True + self._row = [] + elif self.in_target and self.in_row and tag in {"td", "th"}: + self.in_cell = True + self._cell = [] + + def handle_endtag(self, tag: str) -> None: + if self.in_target and self.in_row and tag in {"td", "th"} and self.in_cell: + self._row.append(_clean_text("".join(self._cell))) + self.in_cell = False + elif self.in_target and tag == "tr": + if self._row: + self.rows.append(self._row) + self.in_row = False + elif self.in_target and tag == "table": + self.table_depth -= 1 + if self.table_depth == 0: + self.in_target = False + + def handle_data(self, data: str) -> None: + if self.in_target and self.in_cell: + self._cell.append(data) + + +def _clean_text(value: str) -> str: + return " ".join(html_lib.unescape(value).replace("\xa0", " ").split()) + + +def _parse_number(value: str) -> float | None: + normalized = _clean_text(value).replace(",", "") + if not normalized or normalized in {"....", "-", "—", "N/A"}: + return None + try: + parsed = float(normalized) + except ValueError: + return None + return parsed if math.isfinite(parsed) else None + + +def _parse_period(value: str) -> tuple[date, bool] | None: + match = _PERIOD_RE.fullmatch(_clean_text(value).upper()) + if not match: + return None + month = _MONTHS.get(match.group(1)) + if month is None: + return None + return date(int(match.group(2)), month, 1), bool(match.group(3)) + + +def _parse_published_at(raw_html: str) -> str: + match = _UPDATED_RE.search(_clean_text(raw_html)) + if not match: + raise TourismSourceError("BOT tourism report is missing Last Updated timestamp") + try: + published = datetime.strptime(match.group(1), "%d %b %Y %H:%M").replace(tzinfo=_BANGKOK_TZ) + except ValueError as exc: + raise TourismSourceError("BOT tourism report has an invalid Last Updated timestamp") from exc + return published.isoformat() + + +def _robust_scale(values: list[float]) -> float: + center = median(values) + mad = median([abs(value - center) for value in values]) + if mad > 0: + return max(1.4826 * mad, 0.5) + deviation = pstdev(values) if len(values) > 1 else 0.0 + return max(deviation, 0.5) + + +def parse_bot_tourism_report( + raw_html: str, + *, + retrieved_at: str | None = None, + raw_payload_hash: str | None = None, + source_url: str = BOT_TOURISM_URL, + exposures: Iterable[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Parse a frozen BOT report into the platform snapshot contract.""" + + table_parser = _TableParser() + table_parser.feed(raw_html) + if not table_parser.rows: + raise TourismSourceError("BOT tourism table was not found") + + header = next((row for row in table_parser.rows if any(_parse_period(cell) for cell in row)), None) + tourism_row = next( + ( + row + for row in table_parser.rows + if len(row) > 1 and "number of foreign tourists visiting thailand" in row[1].lower() + ), + None, + ) + if header is None or tourism_row is None: + raise TourismSourceError("BOT tourism table is missing tourism header or tourism row") + + values_by_period: dict[date, tuple[float, bool]] = {} + for label, raw_value in zip(header[2:], tourism_row[2:]): + parsed_period = _parse_period(label) + parsed_value = _parse_number(raw_value) + if parsed_period is not None and parsed_value is not None: + values_by_period[parsed_period[0]] = (parsed_value, parsed_period[1]) + if not values_by_period: + raise TourismSourceError("BOT tourism table contains no numeric tourism observations") + + yoy_points: list[tuple[date, float, bool]] = [] + for period in sorted(values_by_period): + prior_period = date(period.year - 1, period.month, 1) + if prior_period not in values_by_period or values_by_period[prior_period][0] == 0: + continue + current_value, provisional = values_by_period[period] + prior_value = values_by_period[prior_period][0] + yoy = (current_value / prior_value - 1.0) * 100.0 + if math.isfinite(yoy): + yoy_points.append((period, yoy, provisional)) + if len(yoy_points) < 13: + raise TourismSourceError("BOT tourism report needs at least 13 year-over-year points") + + current_period, current_yoy, current_provisional = yoy_points[-1] + history = [point[1] for point in yoy_points[:-1]][-12:] + expected = median(history) + scale = _robust_scale(history) + retrieved = retrieved_at or datetime.now(timezone.utc).isoformat() + raw_hash = raw_payload_hash or hashlib.sha256(raw_html.encode("utf-8")).hexdigest() + published_at = _parse_published_at(raw_html) + vintage_id = f"{BOT_SOURCE_ID}-{published_at[:10]}-{raw_hash[:12]}" + source_meta = { + "source_id": BOT_SOURCE_ID, + "source_url": source_url, + "published_at": published_at, + "retrieved_at": retrieved, + "vintage_id": vintage_id, + "raw_payload_hash": raw_hash, + "parser_version": PARSER_VERSION, + "release_status": "provisional" if current_provisional else "final", + "available_periods": len(values_by_period), + "history_points": len(history), + "raw_snapshot_file": f"{vintage_id}.html", + "snapshot_file": f"{vintage_id}.json", + } + observation = { + "metric_key": "foreign_arrivals_yoy", + "value": round(current_yoy, 4), + "expected": round(expected, 4), + "scale": round(scale, 4), + "unit": "percent", + "period": current_period.strftime("%Y-%m"), + "history_points": len(history), + "provisional": current_provisional, + } + return { + "as_of": date(current_period.year, current_period.month, monthrange(current_period.year, current_period.month)[1]).isoformat(), + "strategy_version": "tourism-v0.2-bot", + "theme": "tourism", + "source": source_meta, + "data_quality": "provisional" if current_provisional else "high", + "observations": [observation], + "exposures": [dict(item) for item in (exposures or [])], + } + + +def _atomic_write(path: Path, content: str | bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp") + if isinstance(content, bytes): + temporary.write_bytes(content) + else: + temporary.write_text(content, encoding="utf-8") + temporary.replace(path) + + +class BotTourismSource: + """Fetch and persist a replayable BOT Tourism Indicators vintage.""" + + def __init__( + self, + *, + url: str = BOT_TOURISM_URL, + timeout: float = 30.0, + opener: Any | None = None, + clock: Callable[[], datetime] | None = None, + raw_dir: Path | None = None, + snapshot_dir: Path | None = None, + ) -> None: + self.url = url + self.timeout = timeout + self.opener = opener or build_opener(HTTPCookieProcessor(CookieJar())) + self.clock = clock or (lambda: datetime.now(timezone.utc)) + self.raw_dir = raw_dir + self.snapshot_dir = snapshot_dir + + def _open(self, request: Request) -> bytes: + try: + with self.opener.open(request, timeout=self.timeout) as response: + status = getattr(response, "status", 200) + if status >= 400: + raise TourismSourceError(f"BOT tourism source returned HTTP {status}") + return response.read() + except TourismSourceError: + raise + except (HTTPError, URLError, TimeoutError, OSError) as exc: + raise TourismSourceError(f"BOT tourism source request failed: {exc.__class__.__name__}") from exc + + def _form_request(self, form_html: str) -> Request: + parser = _FormParser() + parser.feed(form_html) + if not parser.hidden and not parser.options: + raise TourismSourceError("BOT tourism form controls were not found") + fields = dict(parser.hidden) + fields.update(parser.selected) + years = [value for value in parser.options.get("drpFromYear", []) if re.fullmatch(r"\d{4}xxxx", value)] + if not years: + raise TourismSourceError("BOT tourism form has no start year") + fields.update( + { + "drpPeriod": "MTH", + "drpFromMonth": "xxxx01xx", + "drpFromYear": min(years), + "drpToMonth": parser.selected.get("drpToMonth", ""), + "drpToYear": parser.selected.get("drpToYear", ""), + "btnSubmit": "Submit", + } + ) + action = urljoin(self.url, parser.form_action or self.url) + return Request( + action, + data=urlencode(fields).encode("utf-8"), + headers={"Content-Type": "application/x-www-form-urlencoded", "User-Agent": "SET50-Alternative-Data-Platform/0.1"}, + method="POST", + ) + + def fetch(self, *, exposures: Iterable[dict[str, Any]] | None = None) -> dict[str, Any]: + form_request = Request(self.url, headers={"User-Agent": "SET50-Alternative-Data-Platform/0.1"}) + form_bytes = self._open(form_request) + form_html = form_bytes.decode("utf-8", "replace") + report_bytes = self._open(self._form_request(form_html)) + report_html = report_bytes.decode("utf-8", "replace") + retrieved_at = self.clock().astimezone(timezone.utc).isoformat() + raw_hash = hashlib.sha256(report_bytes).hexdigest() + snapshot = parse_bot_tourism_report( + report_html, + retrieved_at=retrieved_at, + raw_payload_hash=raw_hash, + source_url=self.url, + exposures=exposures, + ) + source_meta = snapshot["source"] + if self.raw_dir is not None: + _atomic_write(self.raw_dir / source_meta["raw_snapshot_file"], report_bytes) + if self.snapshot_dir is not None: + _atomic_write( + self.snapshot_dir / source_meta["snapshot_file"], + json.dumps(snapshot, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + ) + return snapshot diff --git a/backend/app/tourism.py b/backend/app/tourism.py index de3397c..fc76be8 100644 --- a/backend/app/tourism.py +++ b/backend/app/tourism.py @@ -26,6 +26,11 @@ def _data_quality(snapshot: dict[str, Any], observations: list[dict[str, Any]]) source_id = str(source.get("source_id", "")) if source_id.startswith("fixture"): return "fixture" + declared_quality = str(snapshot.get("data_quality", "")).strip().lower() + if declared_quality in {"provisional", "high", "low"}: + return declared_quality + if source.get("release_status") == "provisional": + return "provisional" required_source = {"source_id", "source_url", "published_at", "retrieved_at", "vintage_id"} if not required_source.issubset(source): return "low" diff --git a/backend/tests/fixtures/bot_tourism_form.html b/backend/tests/fixtures/bot_tourism_form.html new file mode 100644 index 0000000..61780ed --- /dev/null +++ b/backend/tests/fixtures/bot_tourism_form.html @@ -0,0 +1,16 @@ + + + +
+ + + + + + + + + +
+ + diff --git a/backend/tests/fixtures/bot_tourism_report.html b/backend/tests/fixtures/bot_tourism_report.html new file mode 100644 index 0000000..f544425 --- /dev/null +++ b/backend/tests/fixtures/bot_tourism_report.html @@ -0,0 +1,13 @@ + + +EC_EI_028_S2 Tourism Indicators 1/ + + Last Updated : 31 Jul 2026 14:30 +
+ + + +
IndicatorJAN 2024 pFEB 2024 pMAR 2024 pAPR 2024 pMAY 2024 pJUN 2024 pJUL 2024 pAUG 2024 pSEP 2024 pOCT 2024 pNOV 2024 pDEC 2024 pJAN 2025 pFEB 2025 pMAR 2025 pAPR 2025 pMAY 2025 pJUN 2025 pJUL 2025 pAUG 2025 pSEP 2025 pOCT 2025 pNOV 2025 pDEC 2025 pJAN 2026 pFEB 2026 pMAR 2026 pAPR 2026 pMAY 2026 pJUN 2026 p
11. Number of foreign tourists visiting Thailand (in thousands)3035.303352.302982.722757.132633.462740.373103.422963.152521.012679.183150.243627.443709.103119.452720.462547.122266.572322.772610.372583.642235.852573.742914.813370.443277.913263.802775.202368.902346.851841.55
+
+ + diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 9f33fcd..8f8e143 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -1,4 +1,7 @@ +import json +import tempfile import unittest +from pathlib import Path from app import create_app @@ -44,6 +47,53 @@ class ApiTests(unittest.TestCase): self.assertEqual(body["signal_summary"]["long"], 1) self.assertEqual(body["signal_summary"]["short"], 1) + def test_data_health_reports_lineage_and_replayability(self): + response = self.client.get("/api/v1/data-health") + body = response.get_json() + self.assertEqual(response.status_code, 200) + self.assertEqual(body["status"], "fixture") + self.assertEqual(body["source_id"], "fixture.tourism") + self.assertFalse(body["replayable"]) + self.assertIn("vintage_id", body) + + def test_replay_endpoint_returns_frozen_snapshot_result(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + (root / "fixture-1.json").write_text(json.dumps(self.snapshot), encoding="utf-8") + app = create_app({"TESTING": True, "SNAPSHOT": self.snapshot, "SNAPSHOT_DIR": root}) + response = app.test_client().get("/api/v1/replay/tourism?vintage_id=fixture-1") + body = response.get_json() + self.assertEqual(response.status_code, 200) + self.assertEqual(body["vintage_id"], "fixture-1") + self.assertEqual(body["result"]["theme"], "tourism") + self.assertEqual(body["result"]["signals"][0]["symbol"], "AOT") + + def test_replay_endpoint_rejects_path_traversal(self): + response = self.client.get("/api/v1/replay/tourism?vintage_id=../secret") + self.assertEqual(response.status_code, 400) + + def test_bot_source_mode_uses_injected_adapter(self): + class FakeAdapter: + def fetch(self, *, exposures): + snapshot = dict(self_snapshot) + snapshot["source"] = {**snapshot["source"], "source_id": "bot.ec_ei_028_s2"} + snapshot["data_quality"] = "provisional" + snapshot["exposures"] = exposures + return snapshot + + self_snapshot = self.snapshot + app = create_app( + { + "TESTING": True, + "SNAPSHOT": self.snapshot, + "TOURISM_SOURCE": "bot", + "TOURISM_ADAPTER": FakeAdapter(), + } + ) + body = app.test_client().get("/api/v1/data-health").get_json() + self.assertEqual(body["source_id"], "bot.ec_ei_028_s2") + self.assertEqual(body["status"], "provisional") + def test_paper_ledger_requires_token(self): response = self.client.post( "/api/v1/paper/ledger", diff --git a/backend/tests/test_bot_tourism.py b/backend/tests/test_bot_tourism.py new file mode 100644 index 0000000..84f87c3 --- /dev/null +++ b/backend/tests/test_bot_tourism.py @@ -0,0 +1,99 @@ +import hashlib +import json +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import parse_qs + +from app.bot_tourism import BotTourismSource, TourismSourceError, parse_bot_tourism_report + + +FIXTURES = Path(__file__).parent / "fixtures" +REPORT_HTML = (FIXTURES / "bot_tourism_report.html").read_text(encoding="utf-8") +FORM_HTML = (FIXTURES / "bot_tourism_form.html").read_text(encoding="utf-8") + + +class FakeResponse: + def __init__(self, body): + self.body = body.encode("utf-8") + self.status = 200 + + def read(self): + return self.body + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + +class FakeOpener: + def __init__(self): + self.calls = [] + + def open(self, request, timeout): + body = FORM_HTML if request.data is None else REPORT_HTML + self.calls.append({"url": request.full_url, "body": request.data, "timeout": timeout}) + return FakeResponse(body) + + +class BotTourismSourceTests(unittest.TestCase): + def test_default_source_opener_is_constructible(self): + source = BotTourismSource() + self.assertIsNotNone(source.opener) + + def test_parser_builds_point_in_time_snapshot_from_bot_table(self): + raw_hash = hashlib.sha256(REPORT_HTML.encode("utf-8")).hexdigest() + snapshot = parse_bot_tourism_report( + REPORT_HTML, + retrieved_at="2026-08-23T02:00:00+00:00", + raw_payload_hash=raw_hash, + ) + + self.assertEqual(snapshot["as_of"], "2026-06-30") + self.assertEqual(snapshot["data_quality"], "provisional") + self.assertEqual(snapshot["source"]["source_id"], "bot.ec_ei_028_s2") + self.assertEqual(snapshot["source"]["published_at"], "2026-07-31T14:30:00+07:00") + self.assertEqual(snapshot["source"]["raw_payload_hash"], raw_hash) + observation = snapshot["observations"][0] + self.assertEqual(observation["metric_key"], "foreign_arrivals_yoy") + self.assertEqual(observation["period"], "2026-06") + self.assertLess(observation["value"], 0) + self.assertTrue(observation["scale"] > 0) + self.assertEqual(observation["history_points"], 12) + + def test_parser_rejects_report_without_tourism_row(self): + with self.assertRaisesRegex(TourismSourceError, "tourism table"): + parse_bot_tourism_report("
wrong
") + + def test_source_fetches_history_and_persists_raw_and_normalized_snapshot(self): + opener = FakeOpener() + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source = BotTourismSource( + opener=opener, + clock=lambda: datetime(2026, 8, 23, 2, 0, tzinfo=timezone.utc), + raw_dir=root / "raw", + snapshot_dir=root / "snapshots", + ) + snapshot = source.fetch() + + self.assertEqual(len(opener.calls), 2) + posted = parse_qs(opener.calls[1]["body"].decode("utf-8")) + self.assertEqual(posted["drpFromMonth"], ["xxxx01xx"]) + self.assertEqual(posted["drpFromYear"], ["2015xxxx"]) + self.assertEqual(posted["drpToMonth"], ["xxxx06xx"]) + self.assertEqual(posted["drpToYear"], ["2026xxxx"]) + self.assertEqual(posted["btnSubmit"], ["Submit"]) + + source_meta = snapshot["source"] + raw_path = root / "raw" / source_meta["raw_snapshot_file"] + snapshot_path = root / "snapshots" / source_meta["snapshot_file"] + self.assertEqual(raw_path.read_text(encoding="utf-8"), REPORT_HTML) + self.assertEqual(json.loads(snapshot_path.read_text(encoding="utf-8")), snapshot) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 7d5c504..4981c91 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -3,45 +3,71 @@ ## Project - Path: `/Users/kunthawat/Gitea/set50-alternative-data-platform` +- Branch: `main` +- Current milestone: M1 complete and independently reviewed - Mode: research + paper only - Frontend: Vue 3 + Vite -- Backend: Flask -- Current data: deterministic fixture with lineage metadata +- Backend: Flask `0.2.0` +- Current runtime source: BOT Tourism Indicators (`TOURISM_SOURCE=bot`) ## Completed -- Tourism snapshot schema with source, publication, retrieval and vintage fields. +- Tourism snapshot schema with source, publication, retrieval, vintage and raw hash fields. - Deterministic Tourism Pulse score: standardized surprise × exposure × confidence. +- BOT ASP.NET form adapter with monthly history parsing and trailing 12-point YoY baseline. +- Atomic raw HTML and normalized snapshot persistence under ignored `backend/data/`. +- Read-only `/api/v1/data-health` endpoint. +- Safe `/api/v1/replay/tourism?vintage_id=...` endpoint with identity and path validation. - Ranked target weights and LONG/SHORT/NEUTRAL classification. -- Flask endpoints for health, summary, observations, signals and paper ledger. -- English dashboard with live API data, lineage panel, signal table and paper-entry form. +- English dashboard with live/provisional source label, sign-aware surprise copy and lineage fields. +- HttpOnly paper session and internal paper ledger. - No external webhook or MT5 integration. -## Verified commands +Independent M1 review passed with no concrete security or logic blockers. Non-blocking backlog: add schema-drift, duplicate/reordered-row, and malformed-vintage regression fixtures. + +## Current live vintage ```text -PYTHONPATH=backend .venv/bin/python -m unittest discover -s backend/tests -v -Ran 10 tests ... OK +source_id: bot.ec_ei_028_s2 +published_at: 2026-07-31T14:30:00+07:00 +as_of: 2026-06-30 +status: provisional +available_periods: 138 +history_points: 12 +vintage_id: bot.ec_ei_028_s2-2026-07-31-665981f88b4a +raw_payload_hash: 665981f88b4a30c5bd30026cf1e96279c244ad83725558f4136d952b29756a31 +theme_surprise: -0.35114754 +``` -Paper writes use a server-side token exchange and HttpOnly `paper_session` cookie; the token is not embedded in the frontend bundle. +## Verified commands and live checks + +```text +PYTHONPATH=backend .venv/bin/python -W error -m unittest discover -s backend/tests -v +Ran 18 tests ... OK npm run build Vite build completed successfully. GET /api/v1/health -{"mode":"research","status":"ok","version":"0.1.0"} +HTTP 200; {"mode":"research","status":"ok","version":"0.2.0"} -POST /api/v1/paper/ledger + GET /api/v1/paper/ledger -PAPER_RECORDED and readback verified. +GET /api/v1/data-health +HTTP 200; source_mode=bot, status=provisional, replayable=true -Independent review -PASSED — no concrete security or logic blockers. +GET /api/v1/replay/tourism?vintage_id=bot.ec_ei_028_s2-2026-07-31-665981f88b4a +HTTP 200; replay theme surprise matched live summary exactly. ``` -## Known limitation +Paper writes use a server-side token exchange and HttpOnly `paper_session` cookie; the token is not embedded in the frontend bundle. -The data adapter is a fixture. It demonstrates the contract and calculation, not live tourism data quality. Browser visual screenshot verification was blocked by a Chrome remote-debugging permission prompt; served HTML, API response and frontend build were verified instead. +## Known limitations -## Next action +- BOT data is provisional and may be revised. The vintage/hash contract preserves the fetched version, but this is not a final-data guarantee. +- Only foreign-arrival YoY is live in this slice; occupancy and airport passenger metrics are not yet connected. +- Snapshot storage is local filesystem and single-process; shared persistence is required before multi-worker deployment. +- No investment edge, transaction-cost result, or backtest conclusion has been established. +- Browser screenshot verification remains blocked by the Chrome remote-debugging permission prompt; served HTML/source, live API, fresh Vite build and replay integrity were verified instead. -Implement one replayable Tourism source adapter without changing the snapshot contract. Add parser fixture tests, publication timestamp handling, raw snapshot hash, and a data-health failure state before adding LLM analysis. +## Exact next action + +Validate multiple BOT vintages and add the next real Tourism metric without changing the snapshot contract. Run event-study/backtest checks before adding LLM analysis. diff --git a/docs/engineering-log.md b/docs/engineering-log.md index 21eede5..9a8d72d 100644 --- a/docs/engineering-log.md +++ b/docs/engineering-log.md @@ -4,10 +4,11 @@ | Milestone | Status | Evidence | Next action | |---|---|---|---| -| M0 repo foundation | complete | Flask API, Vue/Vite shell | replace fixture with source adapter | -| Tourism deterministic signal | complete | 7 backend tests pass | add replayable real source | +| M0 repo foundation | complete | Flask API, Vue/Vite shell | keep research/paper guardrails | +| M1 BOT Tourism adapter | complete | 18 tests, live BOT fetch, raw/snapshot persistence | validate multiple vintages | +| Tourism deterministic signal | complete | live foreign-arrivals YoY surprise | add occupancy/airport metric | | Internal paper ledger | complete | POST/readback through live API | persist in PostgreSQL later | -| Dashboard | complete | Vite build + live HTML/API checks | visual browser capture after permission is available | +| Dashboard | complete | Vite build + served source check with live-sign copy | visual browser capture after permission is available | | LLM analysis | deferred | intentionally no LLM dependency in M0 | add after signal lineage is stable | | Webhook receiver | deferred | contract only, no external receiver | choose after core app is usable | | MT5 bridge | deferred | not started | paper bridge after webhook decision | @@ -17,16 +18,20 @@ - Research and paper modes only. - No live orders, external webhook receiver, broker credentials, or MT5 connection. - Deterministic signal is authoritative; LLM will remain downstream. -- Fixture source is clearly marked and must be replaced before investment use. +- Fixture and provisional BOT sources are explicitly labelled; neither is investment-ready without validation. - `target_weight` is recorded in the internal paper ledger; it is not an order. ## Verification -- Backend: 10 unittest tests pass. -- Independent review: **PASSED**; no concrete security or logic blockers. +- Backend: 18 unittest tests pass. +- Independent M1 review: **PASSED**; no concrete security or logic blockers. - Reviewer suggestions: set `PAPER_COOKIE_SECURE=1` outside local HTTP; replace in-memory sessions before multi-worker deployment. +- M1 reviewer backlog: add schema-drift, duplicate/reordered-row, and malformed-vintage regression fixtures. - Frontend: `npm run build` passes with Vite. - Backend health endpoint returns HTTP 200 JSON. - Dashboard served HTML contains the current title, Vue mount point and Vite entry. - Paper ledger POST and readback work through the live API. +- BOT source live fetch parsed 138 monthly periods and persisted raw HTML plus normalized snapshot. +- Data-health reports `source_mode=bot`, `status=provisional`, and `replayable=true`. +- Live vintage replay returned the same theme surprise as the current dashboard summary. - Browser visual capture was blocked by Chrome remote-debugging permission; no permission dialog was clicked. diff --git a/docs/engineering-log/2026-08-23-tourism-bot-adapter.md b/docs/engineering-log/2026-08-23-tourism-bot-adapter.md new file mode 100644 index 0000000..53f17d2 --- /dev/null +++ b/docs/engineering-log/2026-08-23-tourism-bot-adapter.md @@ -0,0 +1,55 @@ +# 2026-08-23 — BOT Tourism source adapter + +## Plan status + +- M1 Tourism source adapter: complete. +- API data-health and replay endpoints: complete. +- Dashboard lineage wiring: complete. +- LLM, webhook receiver, and MT5 bridge: deferred by design. + +## Changed files + +- `backend/app/bot_tourism.py` — BOT ASP.NET form fetcher, table parser, YoY baseline, raw hash, atomic raw/snapshot persistence. +- `backend/app/__init__.py` — `TOURISM_SOURCE=bot`, data-health endpoint, safe vintage replay endpoint, expanded summary lineage. +- `backend/app/tourism.py` — preserves provisional source quality. +- `backend/tests/test_bot_tourism.py` — parser, default opener, fetch form, persistence and failure tests. +- `backend/tests/test_api.py` — source-mode, data-health, replay and path-safety tests. +- `frontend/src/App.vue` — live-vintage lineage fields and sign-aware surprise copy. +- `frontend/src/style.css` — provisional-quality warning badge. +- `README.md` — fixture/BOT run modes and endpoint documentation. + +## Source and live evidence + +- Source: `https://app.bot.or.th/BTWS_STAT/statistics/ReportPage.aspx?reportID=875&language=eng` +- Source ID: `bot.ec_ei_028_s2` +- Published: `2026-07-31T14:30:00+07:00` +- Retrieved: `2026-08-23T03:09:09.490017+00:00` +- As of: `2026-06-30` +- Release status: `provisional` +- Available monthly periods: `138` +- Baseline points: `12` +- Raw payload SHA-256: `665981f88b4a30c5bd30026cf1e96279c244ad83725558f4136d952b29756a31` +- Raw HTML and normalized snapshot were persisted locally under ignored `backend/data/`. + +The latest observation is `foreign_arrivals_yoy` for `2026-06`: value `-9.8495%`, trailing-baseline expectation `-7.2791%`, robust scale `7.32`, resulting in theme surprise `-0.35114754σ`. The dashboard now says **below seasonal expectation** for this live result; it no longer assumes a bullish tourism direction. + +## Verification evidence + +- `PYTHONPATH=backend .venv/bin/python -W error -m unittest discover -s backend/tests -v` — **18 tests passed**. +- `npm run build` — passed. +- BOT adapter live fetch — passed; `138` periods parsed and files persisted. +- `GET /api/v1/health` — HTTP 200, API version `0.2.0`. +- `GET /api/v1/data-health` — HTTP 200, `source_mode=bot`, `status=provisional`, `replayable=true`. +- `GET /api/v1/replay/tourism?vintage_id=bot.ec_ei_028_s2-2026-07-31-665981f88b4a` — HTTP 200; replay surprise matched live summary exactly. +- Served Vite source after restart contains `BOT live vintage`, sign-aware below copy, `Raw hash`, and `warning-tag`. + +## Risks and remaining work + +- BOT report is provisional and may revise. The vintage/hash contract preserves the fetched version, but the strategy must not silently treat provisional data as final. +- Only foreign-arrival YoY is live in this slice; occupancy and airport passenger metrics are still absent. +- Snapshot storage is local filesystem and single-process; shared persistence is required before multi-worker deployment. +- No investment edge, transaction-cost result, or backtest conclusion has been established. + +## Exact next action + +Validate multiple BOT vintages and add the next real Tourism metric (occupancy or airport passenger proxy) without changing the snapshot contract. Run an event-study/backtest before introducing LLM analysis. diff --git a/docs/test-evidence/2026-08-23-tourism-bot-adapter.md b/docs/test-evidence/2026-08-23-tourism-bot-adapter.md new file mode 100644 index 0000000..64ce7c8 --- /dev/null +++ b/docs/test-evidence/2026-08-23-tourism-bot-adapter.md @@ -0,0 +1,41 @@ +# Test evidence — 2026-08-23 BOT Tourism adapter + +## Automated + +```text +PYTHONPATH=backend .venv/bin/python -W error -m unittest discover -s backend/tests -v +Ran 18 tests ... OK + +npm run build +Vite build completed successfully. +``` + +## Live source + +```text +Source: https://app.bot.or.th/BTWS_STAT/statistics/ReportPage.aspx?reportID=875&language=eng +source_id: bot.ec_ei_028_s2 +as_of: 2026-06-30 +status: provisional +available_periods: 138 +history_points: 12 +replayable: true +``` + +## Replay integrity + +The live API summary theme surprise and replay endpoint theme surprise both returned `-0.35114754`. The replay used the persisted vintage identified by `bot.ec_ei_028_s2-2026-07-31-665981f88b4a`. + +## Frontend served verification + +After rebuilding and restarting Vite, the served `/src/App.vue` contained the live-vintage label, sign-aware `below` copy, raw-hash lineage field, and provisional warning class. Browser screenshot capture remains unavailable because the Chrome remote-debugging permission prompt has not been approved. + +## Independent review + +```text +passed: true +security_concerns: [] +logic_errors: [] +``` + +Non-blocking backlog: add schema-drift, duplicate/reordered-row, and malformed-vintage regression fixtures. diff --git a/frontend/src/App.vue b/frontend/src/App.vue index a7f7b0c..99ff432 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -175,7 +175,7 @@ onMounted(loadDashboard)

A first vertical slice from economic observation to explainable portfolio signal.

-
Live dataset
+
{{ summary?.data_health?.source_mode === 'bot' ? 'BOT live vintage' : 'Fixture dataset' }}
As of {{ summary?.as_of || '—' }}
@@ -188,7 +188,7 @@ onMounted(loadDashboard)
Theme surprise
{{ summary.theme_surprise > 0 ? '+' : '' }}{{ formatNumber(summary.theme_surprise) }}σ
-
Above seasonal expectation
+
{{ summary.theme_surprise >= 0 ? 'Above' : 'Below' }} seasonal expectation
Active signals
@@ -218,7 +218,7 @@ onMounted(loadDashboard)
{{ formatNumber(summary.theme_surprise) }}σ - The tourism basket is running above its seasonal expectation. The score is an input, not an order. + The tourism basket is running {{ summary.theme_surprise >= 0 ? 'above' : 'below' }} its seasonal expectation. The score is an input, not an order.
@@ -235,13 +235,17 @@ onMounted(loadDashboard)
02 / Provenance

Can we trust the input?

- {{ summary.data_health.status }} + {{ summary.data_health.status }}
Source{{ summary.data_health.source_id }}
Published{{ formatDate(summary.data_health.published_at) }}
Retrieved{{ formatDate(summary.data_health.retrieved_at) }}
Vintage{{ summary.data_health.vintage_id }}
+
History{{ summary.data_health.history_points || '—' }} points
+
Parser{{ summary.data_health.parser_version || '—' }}
+
Replay{{ summary.data_health.replayable ? 'Available' : 'Not captured' }}
+
Raw hash{{ summary.data_health.raw_payload_hash ? summary.data_health.raw_payload_hash.slice(0, 16) : '—' }}
Every signal will carry its source, release timestamp and vintage. Revised data never silently rewrites the past.
@@ -278,7 +282,7 @@ onMounted(loadDashboard)
04 / Research note

Read the signal as a thesis.

-

Tourism observations are above expectation, so high-exposure names receive positive scores. The system deliberately stops before execution: a human still needs to review valuation, price-in, liquidity and risk.

+

Tourism observations are {{ summary.theme_surprise >= 0 ? 'above' : 'below' }} expectation, so exposure determines which names receive positive or negative scores. The system deliberately stops before execution: a human still needs to review valuation, price-in, liquidity and risk.

Surprise × Exposure × Confidence
diff --git a/frontend/src/style.css b/frontend/src/style.css index 3f6f5d2..2ae6e92 100644 --- a/frontend/src/style.css +++ b/frontend/src/style.css @@ -117,7 +117,7 @@ tbody tr:hover { background: rgba(255,255,255,.025); } .thesis-panel p, .ledger-panel p { margin-top: 14px; color: var(--muted); font-size: 12px; line-height: 1.8; } .thesis-rule { display: flex; align-items: center; gap: 9px; margin-top: 24px; color: var(--mint); font: 10px 'DM Mono', monospace; } .thesis-rule span { width: 22px; height: 1px; background: var(--mint); } -.fixture-tag { color: var(--amber); border-color: rgba(230,185,108,.3); background: var(--amber-soft); } +.warning-tag { color: var(--amber); border-color: rgba(230,185,108,.3); background: var(--amber-soft); } .neutral-tag { color: var(--amber); border-color: rgba(230,185,108,.25); background: var(--amber-soft); } .auth-form { display: grid; gap: 10px; margin-top: 16px; } .auth-copy { color: var(--muted); font-size: 11px; line-height: 1.65; }