fcc0da9c8d
- add te_thailand collector (TradingEconomics) -> 8 factors: interest rate, business loan growth, consumer credit, household debt/GDP, retail sales YoY, consumer confidence, residential property prices, business confidence; feed banks/retail/consumer_staples/nonbank_finance/property/telecom/healthcare - add thai_trade collector (TradingEconomics external sector) -> exports/ imports/current-account factors (concurrent in-tree work, verified green) - fix sign inversion: theme weights were negative on sign:-1 factors (NPL, inflation, unemployment) so higher NPL/inflation RAISED scores; direction now lives only in factor sign, theme weights positive (regression-locked) - tests: te_thailand parse+direction, value-key resolution contract, dashboard 8-sources, scheduler vintage counts; suite 362 OK
252 lines
13 KiB
Python
252 lines
13 KiB
Python
"""Tests for the in-app data scheduler."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from app.scheduler import AppDataScheduler
|
|
|
|
|
|
class _FakeCache:
|
|
def __init__(self): self.calls = {}
|
|
def fetch_or_stale(self, key, fetcher):
|
|
self.calls[key] = self.calls.get(key, 0) + 1
|
|
return fetcher()
|
|
|
|
|
|
class SchedulerTest(unittest.TestCase):
|
|
def test_refresh_all_calls_collectors(self):
|
|
cache = _FakeCache()
|
|
snap_dir = Path(tempfile.mkdtemp())
|
|
sched = AppDataScheduler(cache, snap_dir, interval_seconds=99999)
|
|
# job fetchers import app.* modules; patch their fetch fns to return dicts
|
|
def fake(modname, fn):
|
|
return lambda: {"ok": True}
|
|
with patch("app.auto_credit.fetch_auto_credit", return_value=type("F", (), {"to_dict": lambda self: {"n": 1}})()), \
|
|
patch("app.auto_npl.fetch_auto_npl", return_value=type("F", (), {"to_dict": lambda self: {"n": 2}})()), \
|
|
patch("app.energy_thai.fetch_energy_thai", return_value=type("F", (), {"to_dict": lambda self: {"n": 3}})()), \
|
|
patch("app.macro_thai.fetch_macro_thai", return_value=type("F", (), {"to_dict": lambda self: {"n": 4}})()):
|
|
res = sched.refresh_all()
|
|
# tourism job is bot_tourism.BotTourismSource().fetch — not trivially patched; allow it to fail gracefully
|
|
self.assertGreaterEqual(len(res), 4)
|
|
self.assertTrue(any(r["ok"] for r in res))
|
|
|
|
def test_marker_written(self):
|
|
cache = _FakeCache()
|
|
snap_dir = Path(tempfile.mkdtemp())
|
|
sched = AppDataScheduler(cache, snap_dir, interval_seconds=99999)
|
|
with patch("app.macro_thai.fetch_macro_thai", return_value=type("F", (), {"to_dict": lambda self: {"n": 1}})()):
|
|
sched.refresh_all()
|
|
marker = snap_dir / "scheduler" / "last_refresh.json"
|
|
self.assertTrue(marker.exists())
|
|
import json
|
|
data = json.loads(marker.read_text())
|
|
self.assertIn("at", data)
|
|
|
|
|
|
class VintageCollectionTest(unittest.TestCase):
|
|
"""Automatic PIT vintage collection via the scheduler (offline, deterministic)."""
|
|
|
|
def _fetched(self):
|
|
# a fetched_by_module dict keyed by FACTORS.fetch module holding values
|
|
return {
|
|
"macro_thai": {
|
|
"tourists_ytd_mn": 15.0, "private_consumption_yoy": 3.0,
|
|
"private_investment_yoy": 4.0, "manufacturing_yoy": 1.0,
|
|
"headline_inflation_yoy": 2.0,
|
|
"core_inflation_yoy": 1.2, "unemployment_pct": 1.1,
|
|
"periods": {"private_consumption_yoy": "Jun 2026"},
|
|
},
|
|
"auto_credit": {"new_car_sales_yoy": 5.0, "vehicle_production": 100000.0,
|
|
"auto_exports": 80000.0},
|
|
"auto_npl": {"pct_of_npls": 3.0},
|
|
"bank_npl": {"pct_of_npls": 0.8},
|
|
"energy_thai": {"quarterly": {"q1": {"net_profit": 500.0, "sales": 10000.0}}},
|
|
"thai_trade": {"current_account_usdm": 500.0, "exports_usdm": 34000.0,
|
|
"imports_usdm": 38000.0},
|
|
"te_thailand": {"interest_rate_pct": 1.0, "loans_to_fin_corp": 10000000.0,
|
|
"consumer_credit_thbmn": 5000000.0,
|
|
"household_debt_gdp_pct": 85.0,
|
|
"retail_sales_yoy": 0.0, "consumer_confidence": 50.0,
|
|
"property_prices_yoy": 1.0, "business_confidence": 50.0},
|
|
}
|
|
|
|
def test_pit_factor_vintages_written_on_refresh(self):
|
|
snap_dir = Path(tempfile.mkdtemp())
|
|
sched = AppDataScheduler(_FakeCache(), snap_dir, interval_seconds=99999)
|
|
# avoid the siamchart + dividend network paths touching real data dir:
|
|
with patch("app.scheduler.AppDataScheduler._record_siamchart_vintages", return_value=None), \
|
|
patch("app.scheduler.AppDataScheduler._maybe_refresh_dated_dividends", return_value=None):
|
|
# invoke the PIT-factor recorder directly with a fetched dict
|
|
sched._record_pit_factor_vintages(self._fetched())
|
|
store_dir = snap_dir / "factor_vintages"
|
|
files = list(store_dir.glob("*.jsonl"))
|
|
# every registry factor with a fetch key should have a vintage file
|
|
from app import factors as factors_mod
|
|
expect = [f for f in factors_mod.FACTORS if factors_mod.FACTORS[f].get("fetch")]
|
|
self.assertGreaterEqual(len(files), len(expect) - 1) # some may be skipped if val None
|
|
# verify macro_consumption has a PIT vintage readable by the store
|
|
from app.factor_vintages import FactorVintageStore
|
|
st = FactorVintageStore(snap_dir)
|
|
import datetime as _dt
|
|
now_iso = _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds")
|
|
val = st.value_at("macro_consumption", now_iso)
|
|
self.assertEqual(val, 3.0) # private_consumption_yoy
|
|
|
|
def test_pit_vintage_dedupes_unchanged_value(self):
|
|
snap_dir = Path(tempfile.mkdtemp())
|
|
sched = AppDataScheduler(_FakeCache(), snap_dir, interval_seconds=99999)
|
|
with patch("app.scheduler.AppDataScheduler._record_siamchart_vintages", return_value=None), \
|
|
patch("app.scheduler.AppDataScheduler._maybe_refresh_dated_dividends", return_value=None):
|
|
sched._record_pit_factor_vintages(self._fetched())
|
|
sched._record_pit_factor_vintages(self._fetched()) # same values -> no new points
|
|
from app.factor_vintages import FactorVintageStore
|
|
st = FactorVintageStore(snap_dir)
|
|
rows = st.series("macro_consumption")
|
|
self.assertEqual(len(rows), 1) # unchanged value not re-appended
|
|
|
|
def test_siamchart_vintage_persisted_when_snapshot_present(self):
|
|
snap_dir = Path(tempfile.mkdtemp())
|
|
# create a snapshot file where the scheduler looks for it
|
|
(snap_dir / "siamchart").mkdir(parents=True, exist_ok=True)
|
|
snap = {"retrieved_at": "2026-06-01T09:00:00+07:00", "rows": [], "details": {}}
|
|
(snap_dir / "siamchart" / "set50_master.json").write_text(
|
|
__import__("json").dumps(snap), encoding="utf-8")
|
|
sched = AppDataScheduler(_FakeCache(), snap_dir, interval_seconds=99999)
|
|
with patch("app.scheduler.AppDataScheduler._record_pit_factor_vintages", return_value=None), \
|
|
patch("app.scheduler.AppDataScheduler._maybe_refresh_dated_dividends", return_value=None):
|
|
sched._record_siamchart_vintages()
|
|
manifest = snap_dir / "siamchart_vintages" / "_manifest.json"
|
|
self.assertTrue(manifest.exists())
|
|
import json
|
|
m = json.loads(manifest.read_text())
|
|
self.assertEqual(len(m.get("snapshots", {})), 1)
|
|
|
|
def test_siamchart_vintage_noop_without_snapshot(self):
|
|
snap_dir = Path(tempfile.mkdtemp())
|
|
sched = AppDataScheduler(_FakeCache(), snap_dir, interval_seconds=99999)
|
|
with patch("app.scheduler.AppDataScheduler._record_pit_factor_vintages", return_value=None), \
|
|
patch("app.scheduler.AppDataScheduler._maybe_refresh_dated_dividends", return_value=None):
|
|
sched._record_siamchart_vintages()
|
|
manifest = snap_dir / "siamchart_vintages" / "_manifest.json"
|
|
self.assertFalse(manifest.exists()) if not manifest.exists() else self.assertEqual(
|
|
len(__import__("json").loads(manifest.read_text()).get("snapshots", {})), 1)
|
|
|
|
|
|
class CadenceAndHealthTest(unittest.TestCase):
|
|
"""Per-source cadence + source-health log (user requirements)."""
|
|
|
|
def test_job_due_respects_frequency_marker(self):
|
|
import json
|
|
snap_dir = Path(tempfile.mkdtemp())
|
|
sched = AppDataScheduler(_FakeCache(), snap_dir, interval_seconds=99999)
|
|
job = {"key": "macro_thai", "frequency": "monthly"}
|
|
# never run -> due
|
|
self.assertTrue(sched._job_due(job))
|
|
sched._mark_job_run(job)
|
|
# just marked -> not due again for ~30 days
|
|
self.assertFalse(sched._job_due(job))
|
|
# age the marker back 40 days -> due again
|
|
import datetime as _dt
|
|
old = (_dt.datetime.now().astimezone() - _dt.timedelta(days=40)).isoformat(timespec="seconds")
|
|
(snap_dir / "scheduler" / "job_macro_thai.json").write_text(
|
|
json.dumps({"at": old}), encoding="utf-8")
|
|
self.assertTrue(sched._job_due(job))
|
|
|
|
def test_analyze_error_classifies(self):
|
|
from app.scheduler import AppDataScheduler
|
|
self.assertEqual(AppDataScheduler._analyze_error("timed out connecting"), "timeout")
|
|
self.assertEqual(AppDataScheduler._analyze_error("Connection refused to host"), "network")
|
|
self.assertEqual(AppDataScheduler._analyze_error("HTTP 404 Not Found"), "http")
|
|
self.assertEqual(AppDataScheduler._analyze_error("JSONDecodeError: expecting value"), "parse")
|
|
self.assertEqual(AppDataScheduler._analyze_error("page structure changed - KeyError 'field'"), "structure")
|
|
self.assertEqual(AppDataScheduler._analyze_error("unauthorized token expired"), "auth")
|
|
self.assertEqual(AppDataScheduler._analyze_error(""), "other")
|
|
|
|
def test_source_health_log_written_and_readable(self):
|
|
import json
|
|
snap_dir = Path(tempfile.mkdtemp())
|
|
sched = AppDataScheduler(_FakeCache(), snap_dir, interval_seconds=99999)
|
|
results = [
|
|
{"key": "macro_thai", "label": "ภาพรวม (BOT)", "ok": True, "at": "2026-08-28T00:00:00+07:00"},
|
|
{"key": "auto_npl", "label": "NPL รถ", "ok": False,
|
|
"error": "JSONDecodeError: expecting value at line 1 (structure change?)",
|
|
"at": "2026-08-28T00:01:00+07:00"},
|
|
]
|
|
sched._append_source_log(results)
|
|
entries = sched._load_source_health()
|
|
self.assertEqual(len(entries), 2)
|
|
by_key = {e["key"]: e for e in entries}
|
|
self.assertTrue(by_key["macro_thai"]["ok"])
|
|
self.assertFalse(by_key["auto_npl"]["ok"])
|
|
self.assertEqual(by_key["auto_npl"]["category"], "parse")
|
|
|
|
def test_refresh_only_runs_due_jobs(self):
|
|
# with frequency markers set to "now", a second refresh_all in the same
|
|
# tick should skip all factor jobs but still try daily jobs.
|
|
snap_dir = Path(tempfile.mkdtemp())
|
|
sched = AppDataScheduler(_FakeCache(), snap_dir, interval_seconds=99999)
|
|
from app.scheduler import _REFRESH_JOBS, _DAILY_JOBS
|
|
for job in _REFRESH_JOBS + _DAILY_JOBS:
|
|
sched._mark_job_run(job)
|
|
# all marked -> a refresh tick runs nothing successfully (no network)
|
|
results = sched.refresh_all()
|
|
# daily jobs that ARE collections we stubbed: none should hard-crash
|
|
self.assertIsInstance(results, list)
|
|
|
|
|
|
class DividendCooldownTest(unittest.TestCase):
|
|
def _sched(self, snap_dir, cooldown):
|
|
return AppDataScheduler(_FakeCache(), snap_dir, interval_seconds=99999,
|
|
dividend_cooldown_seconds=cooldown)
|
|
|
|
def test_refreshes_once_then_respects_cooldown(self):
|
|
snap_dir = Path(tempfile.mkdtemp())
|
|
sched = self._sched(snap_dir, cooldown=3600) # 1h cooldown
|
|
calls = {"n": 0}
|
|
|
|
def fake_fetch(sym):
|
|
calls["n"] += 1
|
|
return [("2025-01-01", 1.0)]
|
|
|
|
fake_fv = {"factors": [{"symbol": "PTT"}]}
|
|
with patch("app.siamchart_factors.build_factor_view", return_value=fake_fv), \
|
|
patch("app.siamchart.fetch_dividend_history", side_effect=fake_fetch):
|
|
sched._maybe_refresh_dated_dividends() # first -> fetch
|
|
n_after_first = calls["n"]
|
|
sched._maybe_refresh_dated_dividends() # second, within cooldown -> skip
|
|
self.assertEqual(n_after_first, 1) # fetched once
|
|
self.assertEqual(calls["n"], 1) # second call skipped
|
|
self.assertTrue((snap_dir / "dividends" / "ledger.json").exists())
|
|
|
|
def test_refetches_after_cooldown_elapses(self):
|
|
snap_dir = Path(tempfile.mkdtemp())
|
|
sched = self._sched(snap_dir, cooldown=1) # 1s cooldown
|
|
calls = {"n": 0}
|
|
|
|
def fake_fetch(sym):
|
|
calls["n"] += 1
|
|
return [("2025-01-01", 1.0)]
|
|
|
|
fake_fv = {"factors": [{"symbol": "PTT"}]}
|
|
with patch("app.siamchart_factors.build_factor_view", return_value=fake_fv), \
|
|
patch("app.siamchart.fetch_dividend_history", side_effect=fake_fetch):
|
|
sched._maybe_refresh_dated_dividends()
|
|
self.assertEqual(calls["n"], 1)
|
|
# force cooldown to have elapsed by rewriting the marker back in time
|
|
import json
|
|
import datetime as _dt
|
|
sched._div_marker.write_text(
|
|
json.dumps({"at": (_dt.datetime.now().astimezone() - _dt.timedelta(hours=1)).isoformat()}),
|
|
encoding="utf-8",
|
|
)
|
|
sched._maybe_refresh_dated_dividends()
|
|
self.assertEqual(calls["n"], 2) # refetched
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|