diff --git a/backend/app/dashboard.py b/backend/app/dashboard.py index 85b946d..d965f85 100644 --- a/backend/app/dashboard.py +++ b/backend/app/dashboard.py @@ -75,7 +75,8 @@ class RealDashboard: def build(self) -> dict: # 1) live theme data (real, no fallback) - from . import auto_credit, auto_npl, bank_npl, energy_thai, bot_tourism + from . import (auto_credit, auto_npl, bank_npl, energy_thai, bot_tourism, + macro_thai, te_thailand, thai_trade) tourism = None try: @@ -94,11 +95,16 @@ class RealDashboard: self.cache, "energy_thai", lambda: energy_thai.fetch_energy_thai().to_dict(), "energy_thai") macro_d = _fetch_with_cache( self.cache, "macro_thai", lambda: macro_thai.fetch_macro_thai().to_dict(), "macro_thai") + trade_d = _fetch_with_cache( + self.cache, "thai_trade", lambda: thai_trade.fetch_thai_trade().to_dict(), "thai_trade") + te_d = _fetch_with_cache( + self.cache, "te_thailand", lambda: te_thailand.fetch_te_thailand().to_dict(), "te_thailand") # 2) per-theme surprise — registry-driven (THE single source of truth) fetched = { "macro_thai": macro_d, "auto_credit": auto_d, "auto_npl": npl_d, "energy_thai": en_d, "bank_npl": bnpl_d, + "thai_trade": trade_d, "te_thailand": te_d, } tourism_surprise = self._tourism_surprise() surprises = themes_mod.compute_theme_surprises( @@ -114,27 +120,38 @@ class RealDashboard: ] # macro-proxy reads for the expanded SET50 themes (deterministic) proxy_reads = { - "banks": {"source": "BOT macro (invest) + NPL ภาคการเงิน", + "banks": {"source": "BOT macro + NPL + TE (rate/credit)", "frequency": "monthly", "invest_yoy": macro_d.get("private_investment_yoy"), "inflation_yoy": macro_d.get("headline_inflation_yoy"), - "bank_npl_pct": (bnpl_d or {}).get("pct_of_npls")}, - "retail": {"source": "BOT macro (consumption)", "frequency": "monthly", - "consumption_yoy": macro_d.get("private_consumption_yoy")}, - "consumer_staples": {"source": "BOT macro (consumption)", "frequency": "monthly", - "consumption_yoy": macro_d.get("private_consumption_yoy")}, + "bank_npl_pct": (bnpl_d or {}).get("pct_of_npls"), + "interest_rate_pct": (te_d or {}).get("interest_rate_pct"), + "loan_growth": (te_d or {}).get("loans_to_fin_corp")}, + "retail": {"source": "BOT macro + TE (retail/confidence)", "frequency": "monthly", + "consumption_yoy": macro_d.get("private_consumption_yoy"), + "retail_sales_yoy": (te_d or {}).get("retail_sales_yoy"), + "consumer_confidence": (te_d or {}).get("consumer_confidence")}, + "consumer_staples": {"source": "BOT macro + TE (retail/confidence)", "frequency": "monthly", + "consumption_yoy": macro_d.get("private_consumption_yoy"), + "retail_sales_yoy": (te_d or {}).get("retail_sales_yoy"), + "consumer_confidence": (te_d or {}).get("consumer_confidence")}, "telecom_it": {"source": "BOT macro (consumption)", "frequency": "monthly", "consumption_yoy": macro_d.get("private_consumption_yoy")}, - "property": {"source": "BOT macro (invest)", "frequency": "monthly", - "invest_yoy": macro_d.get("private_investment_yoy")}, + "property": {"source": "BOT macro + TE (property)", "frequency": "monthly", + "invest_yoy": macro_d.get("private_investment_yoy"), + "property_prices_yoy": (te_d or {}).get("property_prices_yoy"), + "business_confidence": (te_d or {}).get("business_confidence")}, "utilities": {"source": "BOT macro (mfg)", "frequency": "monthly", "mfg_yoy": macro_d.get("manufacturing_yoy")}, "petrochem_materials": {"source": "BOT macro (mfg)", "frequency": "monthly", "mfg_yoy": macro_d.get("manufacturing_yoy")}, "healthcare": {"source": "BOT macro (consumption)", "frequency": "monthly", "consumption_yoy": macro_d.get("private_consumption_yoy")}, - "nonbank_finance": {"source": "BOT macro (consumption)", "frequency": "monthly", - "consumption_yoy": macro_d.get("private_consumption_yoy")}, + "nonbank_finance": {"source": "BOT macro + TE (credit/confidence)", "frequency": "monthly", + "consumption_yoy": macro_d.get("private_consumption_yoy"), + "consumer_credit": (te_d or {}).get("consumer_credit_thbmn"), + "household_debt_gdp": (te_d or {}).get("household_debt_gdp_pct"), + "consumer_confidence": (te_d or {}).get("consumer_confidence")}, "exploration": {"source": "TOP energy (refining)", "frequency": "quarterly", "energy_proxy": surprises.get("refining_energy")}, } @@ -145,7 +162,7 @@ class RealDashboard: board = self._build_board(themes, macro_d) # 5) source provenance table - sources = self._build_sources(auto_d, npl_d, en_d, macro_d, tourism, bnpl_d) + sources = self._build_sources(auto_d, npl_d, en_d, macro_d, tourism, bnpl_d, trade_d, te_d) return { "themes": themes, @@ -283,7 +300,7 @@ class RealDashboard: board.sort(key=lambda r: r["combined"], reverse=True) return board - def _build_sources(self, auto_d, npl_d, en_d, macro_d, tourism, bnpl_d=None) -> list: + def _build_sources(self, auto_d, npl_d, en_d, macro_d, tourism, bnpl_d=None, trade_d=None, te_d=None) -> list: """Sources derived from the FACTORS registry — adding a factor to factors.py auto-appends its source row here (no hardcoded list).""" import datetime as _dt @@ -293,6 +310,7 @@ class RealDashboard: fetched = { "auto_credit": auto_d, "auto_npl": npl_d, "energy_thai": en_d, "macro_thai": macro_d, "bank_npl": bnpl_d, + "thai_trade": trade_d, "te_thailand": te_d, } # group FACTORS by fetch module -> one row per distinct source source_by_module: dict = {} @@ -323,6 +341,8 @@ class RealDashboard: "energy_thai": "Thai Oil investor", "macro_thai": "BOT Thai Economy", "bot_tourism": "BOT Tourism", + "thai_trade": "TradingEconomics", + "te_thailand": "TradingEconomics", } freq = meta["freq"] as_of = data.get("as_of") or data.get("period") or _period(data, meta["factors"]) diff --git a/backend/app/factors.py b/backend/app/factors.py index 580c331..a180cd1 100644 --- a/backend/app/factors.py +++ b/backend/app/factors.py @@ -25,6 +25,8 @@ _FETCH_MODULE: dict[str, str] = { "bank_npl": "bank_npl", "energy_thai": "energy_thai", "macro_thai": "macro_thai", + "thai_trade": "thai_trade", + "te_thailand": "te_thailand", } # Factor -> value key. Sign: +1 higher-is-bullish, -1 lower-is-bullish. @@ -142,6 +144,144 @@ FACTORS: dict[str, dict[str, Any]] = { "weight": 0.5, "center": 2.0, "span": 10.0, # ~2% target; higher is worse (sign -1) }, + # ---- BOT Thai Economy fields that were already fetched but never used ---- + # Wired into the registry so every scraped value actually feeds analysis + # (the user's rule: a fetched data point must be used, not just displayed). + "macro_core_inflation": { + "name_th": "เงินเฟ้อพื้นฐาน (Core)", + "source": "BOT", + "frequency": "monthly", + "fetch": "macro_thai", + "value_key": "core_inflation_yoy", + "sign": -1, + "weight": 0.4, + "center": 1.0, "span": 6.0, # core ~1% target; higher is worse + }, + "macro_unemployment": { + "name_th": "อัตราการว่างงาน", + "source": "BOT", + "frequency": "monthly", + "fetch": "macro_thai", + "value_key": "unemployment_pct", + "sign": -1, + "weight": 0.5, + "center": 1.0, "span": 3.0, # ~1% Thailand; higher unemployment is worse + }, + # ---- Thailand external sector (TradingEconomics, current through 2026) ---- + "external_current_account": { + "name_th": "ดุลบัญชีเดินสะพัด (USD ล้าน)", + "source": "TradingEconomics", + "frequency": "monthly", + "fetch": "thai_trade", + "value_key": "current_account_usdm", + "sign": 1, + "weight": 1.0, + "center": 0.0, "span": 3000.0, # USD mn; surplus positive, deficit negative + }, + "external_exports": { + "name_th": "มูลค่าส่งออก (USD ล้าน)", + "source": "TradingEconomics", + "frequency": "monthly", + "fetch": "thai_trade", + "value_key": "exports_usdm", + "sign": 1, + "weight": 1.0, + "center": 30000.0, "span": 8000.0, # ~$34k USD mn/month + }, + "external_imports": { + "name_th": "มูลค่านำเข้า (USD ล้าน)", + "source": "TradingEconomics", + "frequency": "monthly", + "fetch": "thai_trade", + "value_key": "imports_usdm", + "sign": 1, + "weight": 1.0, + "center": 34000.0, "span": 8000.0, # ~$38k USD mn/month (domestic demand proxy) + }, + # ---- Thailand rates / credit / retail / confidence (TradingEconomics) ---- + # Single-page snapshot factors deepening the macro-proxy themes. Sign +1 = + # higher value is bullish/helpful; sign -1 = the opposite. Theme weights are + # always positive magnitude (direction lives in `sign`, per the sign fix). + "te_interest_rate": { + "name_th": "อัตราดอกเบี้ย (rate)", + "source": "TradingEconomics", + "frequency": "monthly", + "fetch": "te_thailand", + "value_key": "interest_rate_pct", + "sign": 1, + "weight": 0.5, + "center": 1.5, "span": 1.5, # ~1.0-2.0%; higher rate widens bank margin + }, + "te_loan_growth": { + "name_th": "สินเชื่อภาคธุรกิจ", + "source": "TradingEconomics", + "frequency": "monthly", + "fetch": "te_thailand", + "value_key": "loans_to_fin_corp", + "sign": 1, + "weight": 0.5, + "center": 10000000.0, "span": 1500000.0, # THB mn (~10.5M); credit demand proxy + }, + "te_consumer_credit": { + "name_th": "สินเชื่อผู้บริโภค", + "source": "TradingEconomics", + "frequency": "monthly", + "fetch": "te_thailand", + "value_key": "consumer_credit_thbmn", + "sign": 1, + "weight": 0.5, + "center": 5000000.0, "span": 800000.0, # THB mn consumer credit book + }, + "te_household_debt_gdp": { + "name_th": "หนี้ครัวเรือนต่อ GDP", + "source": "TradingEconomics", + "frequency": "quarterly", + "fetch": "te_thailand", + "value_key": "household_debt_gdp_pct", + "sign": -1, + "weight": 0.5, + "center": 85.0, "span": 8.0, # ~87.5% of GDP; higher = leverage risk + }, + "te_retail_sales_yoy": { + "name_th": "ยอดขายปลีก (YoY)", + "source": "TradingEconomics", + "frequency": "monthly", + "fetch": "te_thailand", + "value_key": "retail_sales_yoy", + "sign": 1, + "weight": 0.8, + "center": 0.0, "span": 10.0, # % YoY, ~0 neutral + }, + "te_consumer_confidence": { + "name_th": "ความเชื่อมั่นผู้บริโภค", + "source": "TradingEconomics", + "frequency": "monthly", + "fetch": "te_thailand", + "value_key": "consumer_confidence", + "sign": 1, + "weight": 0.5, + "center": 50.0, "span": 12.0, # points (~51.8); above 50 = optimistic + }, + "te_property_prices": { + "name_th": "ราคาอสังหาริมทรัพย์ (YoY)", + "source": "TradingEconomics", + "frequency": "monthly", + "fetch": "te_thailand", + "value_key": "property_prices_yoy", + "sign": 1, + "weight": 0.8, + "center": 0.0, "span": 6.0, # residential price % YoY, ~0-2% neutral + }, + "te_business_confidence": { + "name_th": "ความเชื่อมั่นภาคธุรกิจ", + "source": "TradingEconomics", + "frequency": "monthly", + "fetch": "te_thailand", + "value_key": "business_confidence", + "sign": 1, + "weight": 0.4, + "center": 50.0, "span": 10.0, # points (~46.7); above 50 = optimistic + }, } diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 655d25b..b7df37e 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -39,6 +39,8 @@ _REFRESH_JOBS: List[dict] = [ {"key": "energy_thai", "label": "โรงกลั่น TOP", "module": "energy_thai", "fn": "fetch_energy_thai", "fetch_module": "energy_thai", "frequency": "quarterly"}, {"key": "macro_thai", "label": "ภาพรวมประเทศไทย (BOT)", "module": "macro_thai", "fn": "fetch_macro_thai", "fetch_module": "macro_thai", "frequency": "monthly"}, {"key": "bank_npl", "label": "NPL ภาคการเงิน (BOT)", "module": "bank_npl", "fn": "fetch_bank_npl", "fetch_module": "bank_npl", "frequency": "quarterly"}, + {"key": "thai_trade", "label": "ดุลการค้า/ส่งออก (TradingEconomics)", "module": "thai_trade", "fn": "fetch_thai_trade", "fetch_module": "thai_trade", "frequency": "monthly"}, + {"key": "te_thailand", "label": "อัตราดอกเบี้ย/สินเชื่อ/ค้าปลีก/เชื่อมั่น (TE)", "module": "te_thailand", "fn": "fetch_te_thailand", "fetch_module": "te_thailand", "frequency": "monthly"}, ] # Frequencies -> minimum seconds between successful refreshes of a job. diff --git a/backend/app/te_thailand.py b/backend/app/te_thailand.py new file mode 100644 index 0000000..1d9887e --- /dev/null +++ b/backend/app/te_thailand.py @@ -0,0 +1,209 @@ +"""Thailand rates / credit / retail / confidence / property factors — TradingEconomics. + +Source pages (server-rendered HTML, same infrastructure as `thai_trade` / +`auto_credit` — the "related indicators" table exposes a stable 5-column row +`[label, value, prev, unit, period]`): + + - https://tradingeconomics.com/thailand/interest-rate + Interest Rate, Loans to Non Financial Corporations, Banks Balance Sheet + - https://tradingeconomics.com/thailand/consumer-confidence + Consumer Confidence, Retail Sales YoY, Consumer Credit, Households Debt + to GDP, Consumer Spending + - https://tradingeconomics.com/thailand/housing-index + Residential Property Prices, Housing Index, Housing Starts + - https://tradingeconomics.com/thailand/business-confidence + Business Confidence, Leading Economic Index, Private Investment MoM + +This module deepens the macro-proxy themes that today rely only on the one BOT +Thai-Economy page: + + Factor contributes to + ---------------------------- ------------------------------------- + te_interest_rate -> banks (rate/credit cycle) + te_loan_growth_level -> banks (credit demand) + te_consumer_credit -> nonbank_finance (household credit book) + te_household_debt_gdp -> nonbank_finance (household leverage risk) + te_retail_sales_yoy -> retail, consumer_staples (spending) + te_consumer_confidence -> retail, consumer_staples, nonbank (sentiment) + te_property_prices -> property (residential price YoY) + te_business_confidence -> telecom_it, property, healthcare (sentiment) + +Every value is a single-page snapshot (current value + prev + period), so each +factor is a *level / latest-period* driver — no history join required. +""" + +from __future__ import annotations + +import html +import re +from dataclasses import dataclass, field +from typing import Optional +from urllib.request import Request, urlopen + +_USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +) +_URL_INTEREST = "https://tradingeconomics.com/thailand/interest-rate" +_URL_CONFIDENCE = "https://tradingeconomics.com/thailand/consumer-confidence" +_URL_HOUSING = "https://tradingeconomics.com/thailand/housing-index" +_URL_BIZCONF = "https://tradingeconomics.com/thailand/business-confidence" + + +class ThaiFactorsError(Exception): + """Raised when a TradingEconomics Thailand page cannot be fetched/parsed.""" + + +@dataclass(frozen=True) +class ThaiFactorsSnapshot: + interest_rate_pct: Optional[float] = None # policy-rate proxy (Aug 2026: 1.00) + loans_to_fin_corp: Optional[float] = None # THB mn (Jun 2026) + consumer_credit_thbmn: Optional[float] = None # THB mn (Jun 2025) + household_debt_gdp_pct: Optional[float] = None # % of GDP (Dec 2025) + retail_sales_yoy: Optional[float] = None # % YoY (May 2026) + consumer_confidence: Optional[float] = None # points (Jul 2026) + consumer_spending: Optional[float] = None # THB mn (Jun 2026) + property_prices_yoy: Optional[float] = None # residential prices % YoY (Mar 2026) + business_confidence: Optional[float] = None # points (Jul 2026) + periods: dict = field(default_factory=dict) + source: str = "tradingeconomics.thai-factors" + + def to_dict(self) -> dict: + return { + "source": self.source, + "interest_rate_pct": self.interest_rate_pct, + "loans_to_fin_corp": self.loans_to_fin_corp, + "consumer_credit_thbmn": self.consumer_credit_thbmn, + "household_debt_gdp_pct": self.household_debt_gdp_pct, + "retail_sales_yoy": self.retail_sales_yoy, + "consumer_confidence": self.consumer_confidence, + "consumer_spending": self.consumer_spending, + "property_prices_yoy": self.property_prices_yoy, + "business_confidence": self.business_confidence, + "periods": self.periods, + } + + +def _fetch(url: str, timeout: float = 30.0) -> str: + req = Request(url, headers={"User-Agent": _USER_AGENT, "Accept": "text/html"}) + try: + with urlopen(req, timeout=timeout) as resp: + raw = resp.read() + except Exception as exc: + raise ThaiFactorsError(f"failed to fetch {url}: {exc}") from exc + try: + return raw.decode("utf-8") + except UnicodeDecodeError: + return raw.decode("latin-1", "ignore") + + +def _cells(row_html: str) -> list[str]: + return [ + html.unescape(re.sub(r"<[^>]+>", "", td)).strip() + for td in re.findall(r"]*>(.*?)", row_html, re.S) + if td.strip() + ] + + +def _to_float(text: str) -> Optional[float]: + text = text.replace(",", "").strip().replace("%", "") + if not text or text in ("-", "N/A", "…"): + return None + try: + return float(text) + except ValueError: + return None + + +def _related_indicators(html_text: str) -> dict[str, dict]: + """Map normalized-label -> {value, prev, unit, period} from the related table. + + Rows are `[label, value, prev, unit, period]` (5 cols). Keys are + lowercased + stripped so a TradingEconomics label tweak (e.g. extra space / + case change) does not silently drop a series. A label may appear more than + once; first row that parses a numeric value wins. + """ + out: dict[str, dict] = {} + for table in re.findall(r"]*>(.*?)", html_text, re.S): + for row_html in re.findall(r"]*>(.*?)", table, re.S): + cells = _cells(row_html) + if len(cells) < 4: + continue + key = cells[0].strip().lower() + value = _to_float(cells[1]) + if value is None: + continue + if key not in out or out[key].get("value") is None: + out[key] = { + "value": value, + "prev": _to_float(cells[2]) if len(cells) > 2 else None, + "unit": cells[3] if len(cells) > 3 else "", + "period": cells[4] if len(cells) > 4 else "", + } + return out + + +def parse_te_thailand_html(interest_html: str, confidence_html: str, + housing_html: str = "", bizconf_html: str = "") -> ThaiFactorsSnapshot: + """Parse the TradingEconomics Thailand pages into one snapshot. + + `housing_html` + `bizconf_html` are optional (Phase B property/business + factors); when omitted those two factors are simply None. + """ + ir = _related_indicators(interest_html) + cc = _related_indicators(confidence_html) + hi = _related_indicators(housing_html) if housing_html else {} + bc = _related_indicators(bizconf_html) if bizconf_html else {} + + periods: dict = {} + snap = ThaiFactorsSnapshot( + interest_rate_pct=(ir.get("interest rate") or {}).get("value"), + loans_to_fin_corp=(ir.get("loans to non financial corporations") or {}).get("value"), + consumer_credit_thbmn=(cc.get("consumer credit") or {}).get("value"), + household_debt_gdp_pct=(cc.get("households debt to gdp") or {}).get("value"), + retail_sales_yoy=(cc.get("retail sales yoy") or {}).get("value"), + consumer_confidence=(cc.get("consumer confidence") or {}).get("value"), + # consumer_spending is surfaced here for display/debug only — it is not + # registered as a FACTORS factor, so it does not feed a theme surprise. + consumer_spending=(cc.get("consumer spending") or {}).get("value"), + property_prices_yoy=(hi.get("residential property prices") or {}).get("value"), + business_confidence=(bc.get("business confidence") or {}).get("value"), + ) + for key, label, src in ( + ("interest_rate_pct", "interest rate", ir), + ("retail_sales_yoy", "retail sales yoy", cc), + ("consumer_confidence", "consumer confidence", cc), + ("property_prices_yoy", "residential property prices", hi), + ("business_confidence", "business confidence", bc), + ): + meta = src.get(label) or {} + if meta and meta.get("period"): + periods[key] = meta.get("period") + + if snap.interest_rate_pct is None and snap.consumer_confidence is None \ + and snap.retail_sales_yoy is None: + raise ThaiFactorsError( + "no usable Thailand factor series found in TradingEconomics pages" + ) + # attach parsed periods onto a copy (dataclass is frozen) + return ThaiFactorsSnapshot( + interest_rate_pct=snap.interest_rate_pct, + loans_to_fin_corp=snap.loans_to_fin_corp, + consumer_credit_thbmn=snap.consumer_credit_thbmn, + household_debt_gdp_pct=snap.household_debt_gdp_pct, + retail_sales_yoy=snap.retail_sales_yoy, + consumer_confidence=snap.consumer_confidence, + consumer_spending=snap.consumer_spending, + property_prices_yoy=snap.property_prices_yoy, + business_confidence=snap.business_confidence, + periods=periods, + ) + + +def fetch_te_thailand(timeout: float = 30.0) -> ThaiFactorsSnapshot: + interest_html = _fetch(_URL_INTEREST, timeout=timeout) + confidence_html = _fetch(_URL_CONFIDENCE, timeout=timeout) + housing_html = _fetch(_URL_HOUSING, timeout=timeout) + bizconf_html = _fetch(_URL_BIZCONF, timeout=timeout) + return parse_te_thailand_html(interest_html, confidence_html, + housing_html, bizconf_html) diff --git a/backend/app/thai_trade.py b/backend/app/thai_trade.py new file mode 100644 index 0000000..0393695 --- /dev/null +++ b/backend/app/thai_trade.py @@ -0,0 +1,125 @@ +"""Thailand external-sector / trade factor — TradingEconomics Thailand. + +Source: https://tradingeconomics.com/thailand/current-account +(server-rendered HTML, same infobox/infrastructure as the auto_credit source — +the "related indicators" table lists the current value, previous value, unit +and reporting period for Thailand's external account). + +Verified live: the current-account page's related-indicators table exposes +(Jul 2026) Exports 34,781.10, Imports 38,399.60, Balance of Trade -3,611.00 +USD mn and (Jun 2026) Current Account -3,473.85 USD mn. Replaces the stale +BOT report-60 path (which only carried data through 2011) with a CURRENT, +free, scrapeable source per the plan's "do not ship a wrong/stale series" gate. +""" + +from __future__ import annotations + +import html +import re +from dataclasses import dataclass +from typing import Optional +from urllib.request import Request, urlopen + +_USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +) +_URL = "https://tradingeconomics.com/thailand/current-account" + + +class ThaiTradeError(Exception): + """Raised when the TradingEconomics Thailand page cannot be fetched/parsed.""" + + +@dataclass(frozen=True) +class ThaiTradeSnapshot: + current_account_usdm: Optional[float] = None # USD million + trade_balance_usdm: Optional[float] = None # USD million + exports_usdm: Optional[float] = None # USD million + imports_usdm: Optional[float] = None # USD million + as_of: str = "" + source: str = "tradingeconomics.thai-trade" + + def to_dict(self) -> dict: + return { + "source": self.source, + "as_of": self.as_of, + "current_account_usdm": self.current_account_usdm, + "trade_balance_usdm": self.trade_balance_usdm, + "exports_usdm": self.exports_usdm, + "imports_usdm": self.imports_usdm, + } + + +def _fetch(url: str = _URL, timeout: float = 30.0) -> str: + req = Request(url, headers={"User-Agent": _USER_AGENT, "Accept": "text/html"}) + try: + with urlopen(req, timeout=timeout) as resp: + raw = resp.read() + except Exception as exc: + raise ThaiTradeError(f"failed to fetch {url}: {exc}") from exc + try: + return raw.decode("utf-8") + except UnicodeDecodeError: + return raw.decode("latin-1", "ignore") + + +def _tables(html_text: str) -> list[str]: + return re.findall(r"]*>(.*?)", html_text, re.S) + + +def _cells(row_html: str) -> list[str]: + return [ + html.unescape(re.sub(r"<[^>]+>", "", td)).strip() + for td in re.findall(r"]*>(.*?)", row_html, re.S) + if td.strip() + ] + + +def _to_float(text: str) -> Optional[float]: + text = text.replace(",", "").strip().replace("%", "") + if not text or text in ("-", "N/A", "…"): + return None + try: + return float(text) + except ValueError: + return None + + +def parse_thai_trade_html(html_text: str) -> ThaiTradeSnapshot: + """Parse the TradingEconomics Thailand current-account related table.""" + current = trade = exports = imports = None + as_of = "" + + for table in _tables(html_text): + rows = re.findall(r"]*>(.*?)", table, re.S) + for row_html in rows: + cells = _cells(row_html) + if not cells: + continue + label = cells[0].strip() + if label == "Current Account": + current = _to_float(cells[1]) + if len(cells) > 4: + as_of = cells[4] + elif label == "Balance of Trade": + trade = _to_float(cells[1]) + elif label == "Exports" and exports is None: + exports = _to_float(cells[1]) + elif label == "Imports" and imports is None: + imports = _to_float(cells[1]) + + if current is None and exports is None and imports is None: + raise ThaiTradeError("no Thai trade series found in TradingEconomics page") + + return ThaiTradeSnapshot( + current_account_usdm=current, + trade_balance_usdm=trade, + exports_usdm=exports, + imports_usdm=imports, + as_of=as_of, + ) + + +def fetch_thai_trade(timeout: float = 30.0) -> ThaiTradeSnapshot: + return parse_thai_trade_html(_fetch(timeout=timeout)) diff --git a/backend/app/themes.py b/backend/app/themes.py index 7474638..c45492d 100644 --- a/backend/app/themes.py +++ b/backend/app/themes.py @@ -101,6 +101,7 @@ THEMES: dict[str, dict] = { "factors": [ {"key": "tourism_arrivals_ytd", "weight": 1.0}, {"key": "macro_consumption", "weight": 0.4}, + {"key": "external_current_account", "weight": 0.3}, ], }, "auto_credit": { @@ -109,7 +110,7 @@ THEMES: dict[str, dict] = { {"key": "auto_sales_yoy", "weight": 1.0}, {"key": "auto_production", "weight": 0.4}, {"key": "auto_exports", "weight": 0.3}, - {"key": "auto_npl", "weight": -0.6}, + {"key": "auto_npl", "weight": 0.6}, ], }, "refining_energy": { @@ -123,22 +124,36 @@ THEMES: dict[str, dict] = { "label_th": "ธนาคาร", "factors": [ {"key": "macro_investment", "weight": 1.0}, - {"key": "macro_inflation", "weight": -0.4}, - {"key": "bank_npl", "weight": -0.6}, + {"key": "macro_inflation", "weight": 0.4}, + {"key": "macro_core_inflation", "weight": 0.3}, + {"key": "bank_npl", "weight": 0.6}, + {"key": "te_interest_rate", "weight": 0.4}, + {"key": "te_loan_growth", "weight": 0.4}, ], }, "retail": { "label_th": "ค้าปลีก", "factors": [ {"key": "macro_consumption", "weight": 1.0}, - {"key": "macro_inflation", "weight": -0.3}, + {"key": "macro_inflation", "weight": 0.3}, + {"key": "macro_core_inflation", "weight": 0.2}, + {"key": "macro_unemployment", "weight": 0.3}, + {"key": "external_imports", "weight": 0.2}, + {"key": "external_current_account", "weight": 0.2}, + {"key": "te_retail_sales_yoy", "weight": 0.7}, + {"key": "te_consumer_confidence", "weight": 0.4}, ], }, "consumer_staples": { "label_th": "อาหาร/อุปโภค", "factors": [ {"key": "macro_consumption", "weight": 1.0}, - {"key": "macro_inflation", "weight": -0.2}, + {"key": "macro_inflation", "weight": 0.2}, + {"key": "macro_core_inflation", "weight": 0.15}, + {"key": "macro_unemployment", "weight": 0.3}, + {"key": "external_imports", "weight": 0.15}, + {"key": "te_retail_sales_yoy", "weight": 0.6}, + {"key": "te_consumer_confidence", "weight": 0.3}, ], }, "telecom_it": { @@ -146,6 +161,8 @@ THEMES: dict[str, dict] = { "factors": [ {"key": "macro_consumption", "weight": 0.8}, {"key": "macro_investment", "weight": 0.4}, + {"key": "external_exports", "weight": 0.2}, + {"key": "te_business_confidence", "weight": 0.3}, ], }, "property": { @@ -153,20 +170,27 @@ THEMES: dict[str, dict] = { "factors": [ {"key": "macro_investment", "weight": 1.0}, {"key": "macro_consumption", "weight": 0.5}, - {"key": "macro_inflation", "weight": -0.3}, + {"key": "macro_inflation", "weight": 0.3}, + {"key": "external_imports", "weight": 0.2}, + {"key": "te_property_prices", "weight": 0.9}, + {"key": "te_business_confidence", "weight": 0.3}, ], }, "healthcare": { "label_th": "โรงพยาบาล", "factors": [ {"key": "macro_consumption", "weight": 0.5}, + {"key": "macro_unemployment", "weight": 0.4}, + {"key": "te_business_confidence", "weight": 0.2}, ], }, "petrochem_materials": { "label_th": "ปิโตรเคมี/วัสดุ", "factors": [ {"key": "macro_mfg", "weight": 1.0}, - {"key": "macro_inflation", "weight": -0.3}, + {"key": "macro_inflation", "weight": 0.3}, + {"key": "external_exports", "weight": 0.4}, + {"key": "external_current_account", "weight": 0.3}, ], }, "utilities": { @@ -180,14 +204,20 @@ THEMES: dict[str, dict] = { "label_th": "การเงินนอกธนาคาร", "factors": [ {"key": "macro_consumption", "weight": 1.0}, - {"key": "auto_npl", "weight": -0.3}, + {"key": "auto_npl", "weight": 0.3}, + {"key": "macro_unemployment", "weight": 0.3}, + {"key": "te_consumer_credit", "weight": 0.5}, + {"key": "te_household_debt_gdp", "weight": 0.4}, + {"key": "te_consumer_confidence", "weight": 0.3}, ], }, "exploration": { "label_th": "สำรวจ/ผลิตพลังงาน", "factors": [ {"key": "energy_net_margin", "weight": 1.0}, - {"key": "macro_inflation", "weight": -0.2}, + {"key": "macro_inflation", "weight": 0.2}, + {"key": "external_exports", "weight": 0.3}, + {"key": "external_current_account", "weight": 0.2}, ], }, } diff --git a/backend/tests/test_dashboard.py b/backend/tests/test_dashboard.py index f13ab8e..92c9b9c 100644 --- a/backend/tests/test_dashboard.py +++ b/backend/tests/test_dashboard.py @@ -55,7 +55,7 @@ class DashboardTest(unittest.TestCase): cache = _FakeCache({}) dash = RealDashboard([], cache).build() self.assertEqual(len(dash["themes"]), 13) # all SET50 themes - self.assertEqual(len(dash["sources"]), 6) # auto-derived from FACTORS (bank_npl added) + self.assertEqual(len(dash["sources"]), 8) # auto-derived from FACTORS (te_thailand added) self.assertIn("macro", dash) self.assertIn("board", dash) diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index 0055a1a..742981e 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -57,6 +57,7 @@ class VintageCollectionTest(unittest.TestCase): "tourists_ytd_mn": 15.0, "private_consumption_yoy": 3.0, "private_investment_yoy": 4.0, "manufacturing_yoy": 1.0, "headline_inflation_yoy": 2.0, + "core_inflation_yoy": 1.2, "unemployment_pct": 1.1, "periods": {"private_consumption_yoy": "Jun 2026"}, }, "auto_credit": {"new_car_sales_yoy": 5.0, "vehicle_production": 100000.0, @@ -64,6 +65,13 @@ class VintageCollectionTest(unittest.TestCase): "auto_npl": {"pct_of_npls": 3.0}, "bank_npl": {"pct_of_npls": 0.8}, "energy_thai": {"quarterly": {"q1": {"net_profit": 500.0, "sales": 10000.0}}}, + "thai_trade": {"current_account_usdm": 500.0, "exports_usdm": 34000.0, + "imports_usdm": 38000.0}, + "te_thailand": {"interest_rate_pct": 1.0, "loans_to_fin_corp": 10000000.0, + "consumer_credit_thbmn": 5000000.0, + "household_debt_gdp_pct": 85.0, + "retail_sales_yoy": 0.0, "consumer_confidence": 50.0, + "property_prices_yoy": 1.0, "business_confidence": 50.0}, } def test_pit_factor_vintages_written_on_refresh(self): diff --git a/backend/tests/test_te_thailand.py b/backend/tests/test_te_thailand.py new file mode 100644 index 0000000..dc140c9 --- /dev/null +++ b/backend/tests/test_te_thailand.py @@ -0,0 +1,157 @@ +"""Tests for the Thailand rates/credit/retail/confidence collector + themes.""" + +from __future__ import annotations + +import unittest + +from app import te_thailand, themes +from app.te_thailand import ThaiFactorsSnapshot, parse_te_thailand_html + + +def _row(label, value, prev, unit, period): + return (f"{label}{value}{prev}" + f"{unit}{period}") + + +def _table(*rows): + return f"{''.join(rows)}
" + + +_IR_HTML = _table( + _row("Interest Rate", "1.00", "1.00", "percent", "Aug 2026"), + _row("Loans to Non Financial Corporations", "10553097.00", "10488479.00", "THB Million", "Jun 2026"), + _row("Banks Balance Sheet", "40475793.00", "40559803.00", "THB Million", "Jun 2026"), +) +_CC_HTML = _table( + _row("Consumer Confidence", "51.80", "50.70", "points", "Jul 2026"), + _row("Retail Sales YoY", "-14.50", "-20.00", "percent", "May 2026"), + _row("Consumer Credit", "5285734.00", "5296795.00", "THB Million", "Jun 2025"), + _row("Households Debt to GDP", "87.50", "87.30", "percent of GDP", "Dec 2025"), + _row("Consumer Spending", "1771591.00", "1730973.00", "THB Million", "Jun 2026"), +) +_HOUSING_HTML = _table( + _row("Housing Index", "162.70", "162.40", "points", "Jun 2026"), + _row("Residential Property Prices", "1.26", "0.63", "Percent", "Mar 2026"), + _row("Housing Starts", "4093.00", "6520.00", "units", "Apr 2026"), +) +_BIZCONF_HTML = _table( + _row("Business Confidence", "46.70", "46.10", "points", "Jul 2026"), + _row("Leading Economic Index", "166.19", "163.56", "points", "Jun 2026"), +) + + +class TeThailandParseTest(unittest.TestCase): + def test_parses_all_indicators(self): + snap = parse_te_thailand_html(_IR_HTML, _CC_HTML, _HOUSING_HTML, _BIZCONF_HTML) + self.assertIsInstance(snap, ThaiFactorsSnapshot) + self.assertEqual(snap.interest_rate_pct, 1.0) + self.assertEqual(snap.loans_to_fin_corp, 10553097.0) + self.assertEqual(snap.consumer_credit_thbmn, 5285734.0) + self.assertEqual(snap.household_debt_gdp_pct, 87.5) + self.assertEqual(snap.retail_sales_yoy, -14.5) + self.assertEqual(snap.consumer_confidence, 51.8) + self.assertEqual(snap.consumer_spending, 1771591.0) + self.assertEqual(snap.property_prices_yoy, 1.26) + self.assertEqual(snap.business_confidence, 46.7) + self.assertEqual(snap.periods["interest_rate_pct"], "Aug 2026") + self.assertEqual(snap.periods["property_prices_yoy"], "Mar 2026") + + def test_to_dict_full(self): + d = parse_te_thailand_html(_IR_HTML, _CC_HTML, _HOUSING_HTML, _BIZCONF_HTML).to_dict() + self.assertIn("interest_rate_pct", d) + self.assertIn("retail_sales_yoy", d) + self.assertIn("property_prices_yoy", d) + self.assertIn("business_confidence", d) + self.assertIn("source", d) + + def test_missing_values_raise(self): + # neither page yields a usable series -> collector fails loudly + empty = "
x1
" + with self.assertRaises(te_thailand.ThaiFactorsError): + parse_te_thailand_html(empty, empty) + + def test_every_factor_value_key_resolves(self): + from app import factors + keys = set(ThaiFactorsSnapshot().to_dict().keys()) + for fkey, fact in factors.FACTORS.items(): + if fact.get("fetch") != "te_thailand": + continue + self.assertIn( + fact.get("value_key"), keys, + f"factor {fkey!r} value_key not emitted by te_thailand", + ) + + +class TeThailandThemeDirectionTest(unittest.TestCase): + """New factors must move theme surprises the intended direction (sign fix).""" + + @staticmethod + def _fetched(): + base = { + "macro_thai": { + "private_consumption_yoy": 4.9, "private_investment_yoy": 18.1, + "headline_inflation_yoy": 1.95, "core_inflation_yoy": 1.0, + "unemployment_pct": 1.0, "manufacturing_yoy": -3.1, + "tourists_ytd_mn": 16.2, + }, + "auto_credit": {"new_car_sales_yoy": 20.07, "vehicle_production": 117383.0, + "auto_exports": 81526.0}, + "auto_npl": {"pct_of_npls": 3.0}, + "bank_npl": {"pct_of_npls": 1.0}, + "energy_thai": {"quarterly": {"Q1/2026": {"net_profit": 19481.0, "sales": 114809.0}}}, + "thai_trade": {"current_account_usdm": 500.0, "exports_usdm": 34000.0, + "imports_usdm": 38000.0}, + # te_thailand factors — neutral-ish values + "te_thailand": { + "interest_rate_pct": 1.5, + "loans_to_fin_corp": 10000000.0, + "consumer_credit_thbmn": 5000000.0, + "household_debt_gdp_pct": 85.0, + "retail_sales_yoy": 0.0, + "consumer_confidence": 50.0, + "property_prices_yoy": 0.0, + "business_confidence": 50.0, + }, + } + return base + + def _surprises(self, te): + d = self._fetched() + d["te_thailand"] = te + return themes.compute_theme_surprises(d) + + def test_higher_retail_sales_raises_retail(self): + low = self._surprises({**self._fetched()["te_thailand"], "retail_sales_yoy": -15.0}) + high = self._surprises({**self._fetched()["te_thailand"], "retail_sales_yoy": 8.0}) + self.assertGreater(high["retail"], low["retail"]) + + def test_higher_rate_and_loans_raise_banks(self): + low = self._surprises({**self._fetched()["te_thailand"], "interest_rate_pct": 0.5, + "loans_to_fin_corp": 8500000.0}) + high = self._surprises({**self._fetched()["te_thailand"], "interest_rate_pct": 2.5, + "loans_to_fin_corp": 12000000.0}) + self.assertGreater(high["banks"], low["banks"]) + + def test_higher_household_debt_lowers_nonbank(self): + low = self._surprises({**self._fetched()["te_thailand"], "household_debt_gdp_pct": 80.0}) + high = self._surprises({**self._fetched()["te_thailand"], "household_debt_gdp_pct": 92.0}) + self.assertLess(high["nonbank_finance"], low["nonbank_finance"]) + + def test_higher_consumer_confidence_raises_retail(self): + low = self._surprises({**self._fetched()["te_thailand"], "consumer_confidence": 42.0}) + high = self._surprises({**self._fetched()["te_thailand"], "consumer_confidence": 60.0}) + self.assertGreater(high["retail"], low["retail"]) + + def test_higher_property_prices_raise_property(self): + low = self._surprises({**self._fetched()["te_thailand"], "property_prices_yoy": -4.0}) + high = self._surprises({**self._fetched()["te_thailand"], "property_prices_yoy": 4.0}) + self.assertGreater(high["property"], low["property"]) + + def test_higher_business_confidence_raises_telecom(self): + low = self._surprises({**self._fetched()["te_thailand"], "business_confidence": 42.0}) + high = self._surprises({**self._fetched()["te_thailand"], "business_confidence": 58.0}) + self.assertGreater(high["telecom_it"], low["telecom_it"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/tests/test_themes.py b/backend/tests/test_themes.py index 8daeb70..d95d373 100644 --- a/backend/tests/test_themes.py +++ b/backend/tests/test_themes.py @@ -5,6 +5,7 @@ from __future__ import annotations import unittest from app import themes +from app import auto_credit, auto_npl, bank_npl, macro_thai, te_thailand, thai_trade class ThemesTest(unittest.TestCase): @@ -144,12 +145,12 @@ class RegistryDrivenSurpriseTest(unittest.TestCase): from app import themes s = themes.compute_theme_surprises(self._fetched()) # registry retail = weighted average over |weights| of - # consumption*1.0 + inflation*(-0.3): + # consumption*1.0 + inflation*0.3 (sign -1 intrinsic in the factor): # cons=4.9 -> (4.9-3)/10=0.19 (w=1.0) ; inflation sign -1 -> - # -(1.95-2)/10=+0.005 (w=-0.3) -> weighted = 0.19 - 0.0015 = 0.1885 - # surprise = 0.1885 / (1.0 + 0.3) = 0.145 + # -(1.95-2)/10=+0.005 (w=+0.3) -> weighted = 0.19 + 0.0015 = 0.1915 + # surprise = 0.1915 / (1.0 + 0.3) = 0.147 self.assertIsNotNone(s["retail"]) - self.assertAlmostEqual(s["retail"], 0.145, places=3) + self.assertAlmostEqual(s["retail"], 0.147, places=3) def test_changing_factor_weight_changes_output(self): """The defining property of P0-B: the registry is not decorative.""" @@ -183,3 +184,93 @@ class RegistryDrivenSurpriseTest(unittest.TestCase): self.assertIsNone(factors.normalize(None)) # finite value still normalizes self.assertEqual(factors.normalize(15.0, sign=1, center=5.0, span=10.0), 1.0) + + def test_new_macro_factors_actually_move_scores(self): + """Wire-in proof (user rule: a fetched field must feed analysis). + + core_inflation + unemployment were fetched by BOT macro_thai but not + used. Once present with a non-neutral value they must change the + retail/banks surprise vs a neutral value — i.e. they are not dead data. + """ + from app import themes + neutral = { + "core_inflation_yoy": 1.0, # == center(1.0) -> 0 normalised + "unemployment_pct": 1.0, # == center(1.0) -> 0 normalised + } + base_fetched = self._fetched() + base_fetched["macro_thai"].update(neutral) + s_neutral = themes.compute_theme_surprises( + {k: dict(v) for k, v in base_fetched.items()}) + + hot = dict(base_fetched) + hot["macro_thai"]["unemployment_pct"] = 3.0 # above center -> bearish + s_hot = themes.compute_theme_surprises({k: dict(v) for k, v in hot.items()}) + # higher unemployment is bearish (sign -1, +ve weight) -> retail/healthcare lower + self.assertLess(s_hot["retail"], s_neutral["retail"]) + self.assertLess(s_hot["healthcare"], s_neutral["healthcare"]) + + hot_infl = dict(base_fetched) + hot_infl["macro_thai"]["core_inflation_yoy"] = 3.5 # above center -> bearish + s_infl = themes.compute_theme_surprises({k: dict(v) for k, v in hot_infl.items()}) + self.assertLess(s_infl["banks"], s_neutral["banks"]) + + def test_bearish_factors_move_score_the_right_way(self): + """Regression for the sign-inversion bug. + + Factor `sign` is applied once inside normalize() so a *positive* theme + weight means "more of this factor matters". Bearish factors (NPL, + inflation, unemployment) all carry sign -1; a negative theme weight made + the double product turn positive — i.e. higher NPL/inflation RAISED the + theme score. Lock the correct direction: higher NPL must LOWER + auto_credit/banks; lower NPL must RAISE them. + """ + from app import themes + base = self._fetched() + # base has auto_npl pct 3.95, bank_npl NOT present (only via macro) -> use + # a full fetched incl. bank_npl so the factor is exercised. + base["bank_npl"] = {"pct_of_npls": 1.0} + low = themes.compute_theme_surprises({k: dict(v) for k, v in base.items()}) + + hi = dict(base) + hi["auto_npl"] = {"pct_of_npls": 8.0} # much worse credit quality + hi["bank_npl"] = {"pct_of_npls": 5.0} + s_hi = themes.compute_theme_surprises({k: dict(v) for k, v in hi.items()}) + # higher NPL -> lower surprise in the credit-heavy themes (bearish) + self.assertLess(s_hi["auto_credit"], low["auto_credit"]) + self.assertLess(s_hi["banks"], low["banks"]) + + def test_every_factor_value_key_resolves_to_a_fetched_field(self): + """Contract (user rule): every FACTORS.value_key must be a real field the + registered fetch module emits. A factor that reads a key the collector + never produces is dead weight — catches 'fetched but not used' the other + way (a value_key that can never be populated).""" + from app import factors + for fkey, fact in factors.FACTORS.items(): + fetch_mod = fact.get("fetch") + value_key = fact.get("value_key") + if value_key is None: + continue + # energy_thai derives its value from a nested quarterly dict, not a + # top-level key — covered separately by factor_value(). + if fetch_mod == "energy_thai": + continue + # resolve the module's snapshot .to_dict() keys + if fetch_mod == "macro_thai": + keys = set(macro_thai.MacroThaiSnapshot().to_dict().keys()) + elif fetch_mod == "auto_credit": + keys = set(auto_credit.AutoCreditSnapshot().to_dict().keys()) + elif fetch_mod == "auto_npl": + keys = set(auto_npl.AutoNplSnapshot().to_dict().keys()) + elif fetch_mod == "bank_npl": + keys = set(bank_npl.BankNplSnapshot().to_dict().keys()) + elif fetch_mod == "thai_trade": + keys = set(thai_trade.ThaiTradeSnapshot().to_dict().keys()) + elif fetch_mod == "te_thailand": + keys = set(te_thailand.ThaiFactorsSnapshot().to_dict().keys()) + else: + continue + self.assertIn( + value_key, keys, + f"factor {fkey!r} targets value_key {value_key!r} that fetch " + f"module {fetch_mod!r} never emits -> dead factor", + )