354 lines
14 KiB
Python
354 lines
14 KiB
Python
"""Bank of Thailand Tourism Indicators source adapter."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import html as html_lib
|
|
import json
|
|
import math
|
|
import re
|
|
from calendar import monthrange
|
|
from datetime import date, datetime, timedelta, timezone
|
|
from html.parser import HTMLParser
|
|
from pathlib import Path
|
|
from statistics import median, pstdev
|
|
from typing import Any, Callable, Iterable
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urlencode, urljoin
|
|
from urllib.request import HTTPCookieProcessor, Request, build_opener
|
|
|
|
from http.cookiejar import CookieJar
|
|
|
|
from .vintages import VintageStore
|
|
|
|
BOT_TOURISM_URL = "https://app.bot.or.th/BTWS_STAT/statistics/ReportPage.aspx?reportID=875&language=eng"
|
|
BOT_SOURCE_ID = "bot.ec_ei_028_s2"
|
|
PARSER_VERSION = "bot-tourism-v1"
|
|
_BANGKOK_TZ = timezone(timedelta(hours=7))
|
|
_MONTHS = {name: index for index, name in enumerate(("JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "NOV", "DEC"), start=1)}
|
|
_PERIOD_RE = re.compile(r"^([A-Z]{3})\s+(\d{4})(?:\s+([A-Z]))?$")
|
|
_UPDATED_RE = re.compile(r"Last\s+Updated\s*:\s*(\d{1,2}\s+[A-Za-z]{3}\s+\d{4}\s+\d{2}:\d{2})", re.IGNORECASE)
|
|
|
|
|
|
class TourismSourceError(RuntimeError):
|
|
"""Raised when the external tourism source cannot be trusted or parsed."""
|
|
|
|
|
|
class _FormParser(HTMLParser):
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=True)
|
|
self.in_form = False
|
|
self.form_action = ""
|
|
self.hidden: dict[str, str] = {}
|
|
self.options: dict[str, list[str]] = {}
|
|
self.selected: dict[str, str] = {}
|
|
self._select: str | None = None
|
|
self._select_selected: str | None = None
|
|
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
attributes = dict(attrs)
|
|
if tag == "form":
|
|
self.in_form = True
|
|
self.form_action = attributes.get("action") or ""
|
|
elif self.in_form and tag == "input" and attributes.get("name"):
|
|
if attributes.get("type", "hidden").lower() == "hidden":
|
|
self.hidden[attributes["name"]] = attributes.get("value") or ""
|
|
elif self.in_form and tag == "select" and attributes.get("name"):
|
|
self._select = attributes["name"]
|
|
self.options[self._select] = []
|
|
self._select_selected = None
|
|
elif self.in_form and tag == "option" and self._select:
|
|
value = attributes.get("value") or ""
|
|
self.options[self._select].append(value)
|
|
if "selected" in attributes:
|
|
self._select_selected = value
|
|
|
|
def handle_endtag(self, tag: str) -> None:
|
|
if tag == "select" and self._select:
|
|
values = self.options[self._select]
|
|
self.selected[self._select] = self._select_selected or (values[0] if values else "")
|
|
self._select = None
|
|
self._select_selected = None
|
|
elif tag == "form":
|
|
self.in_form = False
|
|
|
|
|
|
class _TableParser(HTMLParser):
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=True)
|
|
self.in_target = False
|
|
self.table_depth = 0
|
|
self.in_row = False
|
|
self.in_cell = False
|
|
self.rows: list[list[str]] = []
|
|
self._row: list[str] = []
|
|
self._cell: list[str] = []
|
|
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
attributes = dict(attrs)
|
|
if tag == "table" and attributes.get("id") == "dgExcel":
|
|
self.in_target = True
|
|
self.table_depth = 1
|
|
elif self.in_target and tag == "table":
|
|
self.table_depth += 1
|
|
elif self.in_target and tag == "tr":
|
|
self.in_row = True
|
|
self._row = []
|
|
elif self.in_target and self.in_row and tag in {"td", "th"}:
|
|
self.in_cell = True
|
|
self._cell = []
|
|
|
|
def handle_endtag(self, tag: str) -> None:
|
|
if self.in_target and self.in_row and tag in {"td", "th"} and self.in_cell:
|
|
self._row.append(_clean_text("".join(self._cell)))
|
|
self.in_cell = False
|
|
elif self.in_target and tag == "tr":
|
|
if self._row:
|
|
self.rows.append(self._row)
|
|
self.in_row = False
|
|
elif self.in_target and tag == "table":
|
|
self.table_depth -= 1
|
|
if self.table_depth == 0:
|
|
self.in_target = False
|
|
|
|
def handle_data(self, data: str) -> None:
|
|
if self.in_target and self.in_cell:
|
|
self._cell.append(data)
|
|
|
|
|
|
def _clean_text(value: str) -> str:
|
|
return " ".join(html_lib.unescape(value).replace("\xa0", " ").split())
|
|
|
|
|
|
def _parse_number(value: str) -> float | None:
|
|
normalized = _clean_text(value).replace(",", "")
|
|
if not normalized or normalized in {"....", "-", "—", "N/A"}:
|
|
return None
|
|
try:
|
|
parsed = float(normalized)
|
|
except ValueError:
|
|
return None
|
|
return parsed if math.isfinite(parsed) else None
|
|
|
|
|
|
def _parse_period(value: str) -> tuple[date, bool] | None:
|
|
match = _PERIOD_RE.fullmatch(_clean_text(value).upper())
|
|
if not match:
|
|
return None
|
|
month = _MONTHS.get(match.group(1))
|
|
if month is None:
|
|
return None
|
|
return date(int(match.group(2)), month, 1), bool(match.group(3))
|
|
|
|
|
|
def _parse_published_at(raw_html: str) -> str:
|
|
match = _UPDATED_RE.search(_clean_text(raw_html))
|
|
if not match:
|
|
raise TourismSourceError("BOT tourism report is missing Last Updated timestamp")
|
|
try:
|
|
published = datetime.strptime(match.group(1), "%d %b %Y %H:%M").replace(tzinfo=_BANGKOK_TZ)
|
|
except ValueError as exc:
|
|
raise TourismSourceError("BOT tourism report has an invalid Last Updated timestamp") from exc
|
|
return published.isoformat()
|
|
|
|
|
|
def _robust_scale(values: list[float]) -> float:
|
|
center = median(values)
|
|
mad = median([abs(value - center) for value in values])
|
|
if mad > 0:
|
|
return max(1.4826 * mad, 0.5)
|
|
deviation = pstdev(values) if len(values) > 1 else 0.0
|
|
return max(deviation, 0.5)
|
|
|
|
|
|
def parse_bot_tourism_report(
|
|
raw_html: str,
|
|
*,
|
|
retrieved_at: str | None = None,
|
|
raw_payload_hash: str | None = None,
|
|
source_url: str = BOT_TOURISM_URL,
|
|
exposures: Iterable[dict[str, Any]] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Parse a frozen BOT report into the platform snapshot contract."""
|
|
|
|
table_parser = _TableParser()
|
|
table_parser.feed(raw_html)
|
|
if not table_parser.rows:
|
|
raise TourismSourceError("BOT tourism table was not found")
|
|
|
|
header = next((row for row in table_parser.rows if any(_parse_period(cell) for cell in row)), None)
|
|
tourism_row = next(
|
|
(
|
|
row
|
|
for row in table_parser.rows
|
|
if len(row) > 1 and "number of foreign tourists visiting thailand" in row[1].lower()
|
|
),
|
|
None,
|
|
)
|
|
if header is None or tourism_row is None:
|
|
raise TourismSourceError("BOT tourism table is missing tourism header or tourism row")
|
|
|
|
values_by_period: dict[date, tuple[float, bool]] = {}
|
|
for label, raw_value in zip(header[2:], tourism_row[2:]):
|
|
parsed_period = _parse_period(label)
|
|
parsed_value = _parse_number(raw_value)
|
|
if parsed_period is not None and parsed_value is not None:
|
|
values_by_period[parsed_period[0]] = (parsed_value, parsed_period[1])
|
|
if not values_by_period:
|
|
raise TourismSourceError("BOT tourism table contains no numeric tourism observations")
|
|
|
|
yoy_points: list[tuple[date, float, bool]] = []
|
|
for period in sorted(values_by_period):
|
|
prior_period = date(period.year - 1, period.month, 1)
|
|
if prior_period not in values_by_period or values_by_period[prior_period][0] == 0:
|
|
continue
|
|
current_value, provisional = values_by_period[period]
|
|
prior_value = values_by_period[prior_period][0]
|
|
yoy = (current_value / prior_value - 1.0) * 100.0
|
|
if math.isfinite(yoy):
|
|
yoy_points.append((period, yoy, provisional))
|
|
if len(yoy_points) < 13:
|
|
raise TourismSourceError("BOT tourism report needs at least 13 year-over-year points")
|
|
|
|
current_period, current_yoy, current_provisional = yoy_points[-1]
|
|
history = [point[1] for point in yoy_points[:-1]][-12:]
|
|
expected = median(history)
|
|
scale = _robust_scale(history)
|
|
retrieved = retrieved_at or datetime.now(timezone.utc).isoformat()
|
|
raw_hash = raw_payload_hash or hashlib.sha256(raw_html.encode("utf-8")).hexdigest()
|
|
published_at = _parse_published_at(raw_html)
|
|
vintage_id = f"{BOT_SOURCE_ID}-{published_at[:10]}-{raw_hash[:12]}"
|
|
source_meta = {
|
|
"source_id": BOT_SOURCE_ID,
|
|
"source_url": source_url,
|
|
"published_at": published_at,
|
|
"retrieved_at": retrieved,
|
|
"vintage_id": vintage_id,
|
|
"raw_payload_hash": raw_hash,
|
|
"parser_version": PARSER_VERSION,
|
|
"release_status": "provisional" if current_provisional else "final",
|
|
"available_periods": len(values_by_period),
|
|
"history_points": len(history),
|
|
"raw_snapshot_file": f"{vintage_id}.html",
|
|
"snapshot_file": f"{vintage_id}.json",
|
|
}
|
|
observation = {
|
|
"metric_key": "foreign_arrivals_yoy",
|
|
"value": round(current_yoy, 4),
|
|
"expected": round(expected, 4),
|
|
"scale": round(scale, 4),
|
|
"unit": "percent",
|
|
"period": current_period.strftime("%Y-%m"),
|
|
"history_points": len(history),
|
|
"provisional": current_provisional,
|
|
}
|
|
return {
|
|
"as_of": date(current_period.year, current_period.month, monthrange(current_period.year, current_period.month)[1]).isoformat(),
|
|
"strategy_version": "tourism-v0.2-bot",
|
|
"theme": "tourism",
|
|
"source": source_meta,
|
|
"data_quality": "provisional" if current_provisional else "high",
|
|
"observations": [observation],
|
|
"exposures": [dict(item) for item in (exposures or [])],
|
|
}
|
|
|
|
|
|
def _atomic_write(path: Path, content: str | bytes) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_name(f".{path.name}.tmp")
|
|
if isinstance(content, bytes):
|
|
temporary.write_bytes(content)
|
|
else:
|
|
temporary.write_text(content, encoding="utf-8")
|
|
temporary.replace(path)
|
|
|
|
|
|
class BotTourismSource:
|
|
"""Fetch and persist a replayable BOT Tourism Indicators vintage."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
url: str = BOT_TOURISM_URL,
|
|
timeout: float = 30.0,
|
|
opener: Any | None = None,
|
|
clock: Callable[[], datetime] | None = None,
|
|
raw_dir: Path | None = None,
|
|
snapshot_dir: Path | None = None,
|
|
vintage_store: VintageStore | None = None,
|
|
) -> None:
|
|
self.url = url
|
|
self.timeout = timeout
|
|
self.opener = opener or build_opener(HTTPCookieProcessor(CookieJar()))
|
|
self.clock = clock or (lambda: datetime.now(timezone.utc))
|
|
self.raw_dir = raw_dir
|
|
self.snapshot_dir = snapshot_dir
|
|
self.vintage_store = vintage_store
|
|
|
|
def _open(self, request: Request) -> bytes:
|
|
try:
|
|
with self.opener.open(request, timeout=self.timeout) as response:
|
|
status = getattr(response, "status", 200)
|
|
if status >= 400:
|
|
raise TourismSourceError(f"BOT tourism source returned HTTP {status}")
|
|
return response.read()
|
|
except TourismSourceError:
|
|
raise
|
|
except (HTTPError, URLError, TimeoutError, OSError) as exc:
|
|
raise TourismSourceError(f"BOT tourism source request failed: {exc.__class__.__name__}") from exc
|
|
|
|
def _form_request(self, form_html: str) -> Request:
|
|
parser = _FormParser()
|
|
parser.feed(form_html)
|
|
if not parser.hidden and not parser.options:
|
|
raise TourismSourceError("BOT tourism form controls were not found")
|
|
fields = dict(parser.hidden)
|
|
fields.update(parser.selected)
|
|
years = [value for value in parser.options.get("drpFromYear", []) if re.fullmatch(r"\d{4}xxxx", value)]
|
|
if not years:
|
|
raise TourismSourceError("BOT tourism form has no start year")
|
|
fields.update(
|
|
{
|
|
"drpPeriod": "MTH",
|
|
"drpFromMonth": "xxxx01xx",
|
|
"drpFromYear": min(years),
|
|
"drpToMonth": parser.selected.get("drpToMonth", ""),
|
|
"drpToYear": parser.selected.get("drpToYear", ""),
|
|
"btnSubmit": "Submit",
|
|
}
|
|
)
|
|
action = urljoin(self.url, parser.form_action or self.url)
|
|
return Request(
|
|
action,
|
|
data=urlencode(fields).encode("utf-8"),
|
|
headers={"Content-Type": "application/x-www-form-urlencoded", "User-Agent": "SET50-Alternative-Data-Platform/0.1"},
|
|
method="POST",
|
|
)
|
|
|
|
def fetch(self, *, exposures: Iterable[dict[str, Any]] | None = None) -> dict[str, Any]:
|
|
form_request = Request(self.url, headers={"User-Agent": "SET50-Alternative-Data-Platform/0.1"})
|
|
form_bytes = self._open(form_request)
|
|
form_html = form_bytes.decode("utf-8", "replace")
|
|
report_bytes = self._open(self._form_request(form_html))
|
|
report_html = report_bytes.decode("utf-8", "replace")
|
|
retrieved_at = self.clock().astimezone(timezone.utc).isoformat()
|
|
raw_hash = hashlib.sha256(report_bytes).hexdigest()
|
|
snapshot = parse_bot_tourism_report(
|
|
report_html,
|
|
retrieved_at=retrieved_at,
|
|
raw_payload_hash=raw_hash,
|
|
source_url=self.url,
|
|
exposures=exposures,
|
|
)
|
|
if self.vintage_store is not None:
|
|
return self.vintage_store.persist(report_bytes, snapshot)
|
|
source_meta = snapshot["source"]
|
|
if self.raw_dir is not None:
|
|
_atomic_write(self.raw_dir / source_meta["raw_snapshot_file"], report_bytes)
|
|
if self.snapshot_dir is not None:
|
|
_atomic_write(
|
|
self.snapshot_dir / source_meta["snapshot_file"],
|
|
json.dumps(snapshot, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
)
|
|
return snapshot
|