03195dc55d
Automatically keep the real dated dividend ledger fresh inside the app's own refresh loop (this app runs on its own server, independent of Hermes): - backend/app/scheduler.py: AppDataScheduler gained a cooldown-gated _maybe_refresh_dated_dividends() that fetches real dated dividend history (siamchart /stock-info) into data/dividends/ledger.json at most once per dividend_cooldown_seconds (default 6h) — dividend history changes only a few times a year, so we never hammer the source every refresh tick. The fetch is non-fatal: a network failure leaves the previous ledger intact. - backend/app/__init__.py: passes DIVIDEND_REFRESH_COOLDOWN_SECONDS to the scheduler (default 21600s). - tests: cooldown fires once then skips, and refetches after it elapses (2) — full backend 294 passed.
213 lines
9.6 KiB
Python
213 lines
9.6 KiB
Python
"""App-internal background scheduler — refreshes real data on its own schedule.
|
|
|
|
This app runs on its own server (Docker/EasyPanel), independent of Hermes. So
|
|
the periodic data-fetching/analysis must live INSIDE the app. This module runs a
|
|
daemon thread inside the Flask process that:
|
|
|
|
- on an interval (default every 60 min), triggers each collector through the
|
|
shared daily cache (so fresh real data lands as the "current" state),
|
|
- records each successful refresh as a timestamped snapshot under the data dir
|
|
(persistent history the dashboard/sources table can report),
|
|
- logs progress; swallows transient failures (a source being down once must
|
|
not crash the app or the loop).
|
|
|
|
Thread-safety: the loop only calls collectors + writes cache/snapshots; the HTTP
|
|
layer reads the daily cache. No shared mutable in-process state beyond that.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Callable, List, Optional
|
|
|
|
log = logging.getLogger("set50.scheduler")
|
|
|
|
# collectors returning a .to_dict()/dict, keyed by cache key
|
|
# (imported lazily to avoid import cycles at module load)
|
|
# `fetch_module` = the FACTORS.fetch module name this job feeds (for history).
|
|
_REFRESH_JOBS: List[dict] = [
|
|
{"key": "bot_tourism", "label": "ท่องเที่ยว (BOT)", "module": "bot_tourism", "fn": "BotTourismSource().fetch", "fetch_module": "macro_thai"},
|
|
{"key": "auto_credit/tourism", "label": "ยอดขายรถ (TradingEconomics)", "module": "auto_credit", "fn": "fetch_auto_credit", "fetch_module": "auto_credit"},
|
|
{"key": "auto_npl", "label": "NPL รถยนต์ (BOT)", "module": "auto_npl", "fn": "fetch_auto_npl", "fetch_module": "auto_npl"},
|
|
{"key": "energy_thai", "label": "โรงกลั่น TOP", "module": "energy_thai", "fn": "fetch_energy_thai", "fetch_module": "energy_thai"},
|
|
{"key": "macro_thai", "label": "ภาพรวมประเทศไทย (BOT)", "module": "macro_thai", "fn": "fetch_macro_thai", "fetch_module": "macro_thai"},
|
|
{"key": "bank_npl", "label": "NPL ภาคการเงิน (BOT)", "module": "bank_npl", "fn": "fetch_bank_npl", "fetch_module": "bank_npl"},
|
|
]
|
|
|
|
|
|
class AppDataScheduler:
|
|
"""Runs periodic refresh of the real data collectors inside the app."""
|
|
|
|
def __init__(self, cache, data_root: Path, interval_seconds: int = 3600,
|
|
dividend_cooldown_seconds: int = 6 * 3600):
|
|
self.cache = cache
|
|
self.data_root = data_root
|
|
self.interval = max(30, int(interval_seconds)) # never faster than 30s
|
|
self.dividend_cooldown = max(300, int(dividend_cooldown_seconds)) # >=5min
|
|
self._stop = threading.Event()
|
|
self._thread: Optional[threading.Thread] = None
|
|
self._snap_dir = data_root / "scheduler"
|
|
self._snap_dir.mkdir(parents=True, exist_ok=True)
|
|
self._div_marker = self._snap_dir / "dividends_last_refresh.json"
|
|
self._lock = threading.Lock()
|
|
|
|
# -- public lifecycle ---------------------------------------------------
|
|
def start(self) -> None:
|
|
if self._thread and self._thread.is_alive():
|
|
return
|
|
self._stop.clear()
|
|
self._thread = threading.Thread(target=self._loop, name="set50-refresh", daemon=True)
|
|
self._thread.start()
|
|
log.info("set50 data scheduler started (interval=%ss)", self.interval)
|
|
|
|
def stop(self) -> None:
|
|
self._stop.set()
|
|
|
|
# -- internals ----------------------------------------------------------
|
|
def _loop(self) -> None:
|
|
# refresh once shortly after boot (so fresh data is live), then on interval
|
|
self.refresh_all()
|
|
while not self._stop.wait(self.interval):
|
|
try:
|
|
self.refresh_all()
|
|
except Exception: # noqa: BLE001 — loop must survive individual failures
|
|
log.exception("set50 scheduler refresh_all failed (will retry)")
|
|
|
|
def refresh_all(self) -> list[dict]:
|
|
"""Run every collector, warm the daily cache, snapshot the state, and
|
|
append each factor's value to the historical store (P4 enabler)."""
|
|
results: list[dict] = []
|
|
fetched_by_module: dict[str, dict] = {}
|
|
for job in _REFRESH_JOBS:
|
|
res = self._run_job(job)
|
|
results.append(res)
|
|
fetch_module = job.get("fetch_module")
|
|
if res.get("ok") and isinstance(res.get("value"), dict) and fetch_module:
|
|
fetched_by_module[fetch_module] = res["value"]
|
|
self._record_history(fetched_by_module)
|
|
self._maybe_refresh_dated_dividends()
|
|
self._write_marker(results)
|
|
return results
|
|
|
|
def _record_history(self, fetched_by_module: dict[str, dict]) -> None:
|
|
"""Append current factor values to the historical store (append-only).
|
|
|
|
`fetched_by_module` maps FACTORS.fetch module name -> collector dict.
|
|
Runs after each refresh so vintages accumulate; macro/demographic
|
|
factors become learnable (P4) once they have enough history points.
|
|
"""
|
|
try:
|
|
from .factor_history import FactorHistory
|
|
fh = FactorHistory(self.data_root / "factor_history")
|
|
fh.record_all(fetched_by_module)
|
|
except Exception: # noqa: BLE001 — never let history break the refresh loop
|
|
log.exception("set50 factor-history record failed (non-fatal)")
|
|
|
|
def _maybe_refresh_dated_dividends(self) -> None:
|
|
"""Cooldown-gated fetch of real dated dividend history into the ledger.
|
|
|
|
Dividend history changes slowly (a few times a year per name), so we
|
|
only re-fetch at most once per ``dividend_cooldown`` (default 6h), not
|
|
every refresh tick. Non-fatal: a network failure leaves the previous
|
|
ledger intact and just postpones the next update.
|
|
"""
|
|
with self._lock:
|
|
if self._dividends_fresh():
|
|
return
|
|
try:
|
|
from .dividend_ledger import DividendLedger, populate_dated_dividends
|
|
from .siamchart import fetch_dividend_history
|
|
from .siamchart_factors import build_factor_view
|
|
ledger = DividendLedger(self.data_root / "dividends" / "ledger.json")
|
|
fv = build_factor_view()
|
|
symbols = [f["symbol"] for f in fv.get("factors", [])]
|
|
populate_dated_dividends(ledger, symbols, fetch_dividend_history)
|
|
ledger.save()
|
|
self._mark_dividends_refreshed()
|
|
log.info("set50 dated-dividend ledger refreshed (%d symbols)", len(symbols))
|
|
except Exception: # noqa: BLE001 — non-fatal
|
|
log.exception("set50 dated-dividend refresh failed (will retry next cooldown)")
|
|
|
|
def _dividends_fresh(self) -> bool:
|
|
import json
|
|
import datetime as _dt
|
|
if not self._div_marker.is_file():
|
|
return False
|
|
try:
|
|
payload = json.loads(self._div_marker.read_text(encoding="utf-8"))
|
|
last = _dt.datetime.fromisoformat(payload["at"])
|
|
return (_dt.datetime.now().astimezone() - last).total_seconds() < self.dividend_cooldown
|
|
except (OSError, ValueError, KeyError):
|
|
return False
|
|
|
|
def _mark_dividends_refreshed(self) -> None:
|
|
import json
|
|
import datetime as _dt
|
|
try:
|
|
self._div_marker.write_text(
|
|
json.dumps({"at": _dt.datetime.now().astimezone().isoformat(timespec="seconds")}),
|
|
encoding="utf-8",
|
|
)
|
|
except OSError:
|
|
log.exception("could not write dividend refresh marker")
|
|
|
|
def _run_job(self, job: dict) -> dict:
|
|
key = job["key"]
|
|
label = job["label"]
|
|
try:
|
|
value = self.cache.fetch_or_stale(key, self._make_fetcher(job))
|
|
return {"key": key, "label": label, "ok": True, "at": self._now(),
|
|
"value": value}
|
|
except Exception as exc: # noqa: BLE001
|
|
log.warning("set50 refresh failed for %s: %s", key, exc)
|
|
return {"key": key, "label": label, "ok": False, "error": str(exc), "at": self._now()}
|
|
|
|
def _make_fetcher(self, job: dict) -> Callable[[], dict]:
|
|
module_name = job["module"]
|
|
fn = job["fn"]
|
|
|
|
def fetcher() -> dict:
|
|
mod = __import__(f"app.{module_name}", fromlist=["*"])
|
|
obj = mod
|
|
# Support "ClassName().method" (instantiate then call) and plain
|
|
# "func_name" and static object attribute chains.
|
|
parts = fn.split(".")
|
|
for i, part in enumerate(parts):
|
|
is_instantiate = part.endswith("()")
|
|
attr = part[:-2] if is_instantiate else part
|
|
obj = getattr(obj, attr)
|
|
if is_instantiate:
|
|
obj = obj() # construct instance
|
|
if callable(obj):
|
|
value = obj()
|
|
else:
|
|
value = obj
|
|
if hasattr(value, "to_dict"):
|
|
value = value.to_dict()
|
|
return value or {}
|
|
return fetcher
|
|
|
|
def _write_marker(self, results: list[dict]) -> None:
|
|
import json
|
|
import datetime as _dt
|
|
ok = [r for r in results if r["ok"]]
|
|
marker = {
|
|
"at": self._now(),
|
|
"ok": len(ok),
|
|
"total": len(results),
|
|
"sources": results,
|
|
}
|
|
path = self._snap_dir / "last_refresh.json"
|
|
try:
|
|
path.write_text(json.dumps(marker, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
except OSError:
|
|
log.exception("could not write scheduler marker")
|
|
|
|
@staticmethod
|
|
def _now() -> str:
|
|
import datetime as _dt
|
|
return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds")
|