feat(scheduler): per-source cadence + source-health log with failure diagnosis + UI copy

backend/app/__init__.py

@@ -857,6 +857,21 @@ def create_app(config: dict[str, Any] | None = None) -> Flask:
         store = app.extensions["backtest_store"]
         return jsonify({"runs": store.all()})
 
+    @app.get("/api/v1/scheduler/sources")
+    def scheduler_sources():
+        """Source-health log for the data scheduler (refresh cadence + failures).
+
+        Lets the frontend show each source's last outcome and copy the failure
+        detail for diagnosis (e.g. a source that changed its page structure is
+        categorized as `structure`, a network blip as `network`). Empty when no
+        scheduler is wired (e.g. TESTING or a read-only host) or no tick yet.
+        """
+        sched = app.extensions.get("data_scheduler")
+        if sched is None:
+            return jsonify({"sources": []})
+        limit = int(request.args.get("limit", "200"))
+        return jsonify({"sources": sched._load_source_health(limit)})
+
     @app.post("/api/v1/backtest")
     def run_backtest_endpoint():
         """Run a real backtest over [start, end] with capital; persist result."""

backend/app/scheduler.py

@@ -28,13 +28,33 @@ log = logging.getLogger("set50.scheduler")
 # collectors returning a .to_dict()/dict, keyed by cache key
 # (imported lazily to avoid import cycles at module load)
 # `fetch_module` = the FACTORS.fetch module name this job feeds (for history).
+# `frequency`    = natural refresh cadence of the source. A job is only run
+#                  once its cooldown window has elapsed — the strategy rebalances
+#                  a few times a year, so polling slow sources hourly wastes
+#                  resources. Values: daily | weekly | monthly | quarterly.
 _REFRESH_JOBS: List[dict] = [
-    {"key": "bot_tourism", "label": "ท่องเที่ยว (BOT)", "module": "bot_tourism", "fn": "BotTourismSource().fetch", "fetch_module": "macro_thai"},
-    {"key": "auto_credit/tourism", "label": "ยอดขายรถ (TradingEconomics)", "module": "auto_credit", "fn": "fetch_auto_credit", "fetch_module": "auto_credit"},
-    {"key": "auto_npl", "label": "NPL รถยนต์ (BOT)", "module": "auto_npl", "fn": "fetch_auto_npl", "fetch_module": "auto_npl"},
-    {"key": "energy_thai", "label": "โรงกลั่น TOP", "module": "energy_thai", "fn": "fetch_energy_thai", "fetch_module": "energy_thai"},
-    {"key": "macro_thai", "label": "ภาพรวมประเทศไทย (BOT)", "module": "macro_thai", "fn": "fetch_macro_thai", "fetch_module": "macro_thai"},
-    {"key": "bank_npl", "label": "NPL ภาคการเงิน (BOT)", "module": "bank_npl", "fn": "fetch_bank_npl", "fetch_module": "bank_npl"},
+    {"key": "bot_tourism", "label": "ท่องเที่ยว (BOT)", "module": "bot_tourism", "fn": "BotTourismSource().fetch", "fetch_module": "macro_thai", "frequency": "monthly"},
+    {"key": "auto_credit/tourism", "label": "ยอดขายรถ (TradingEconomics)", "module": "auto_credit", "fn": "fetch_auto_credit", "fetch_module": "auto_credit", "frequency": "monthly"},
+    {"key": "auto_npl", "label": "NPL รถยนต์ (BOT)", "module": "auto_npl", "fn": "fetch_auto_npl", "fetch_module": "auto_npl", "frequency": "quarterly"},
+    {"key": "energy_thai", "label": "โรงกลั่น TOP", "module": "energy_thai", "fn": "fetch_energy_thai", "fetch_module": "energy_thai", "frequency": "quarterly"},
+    {"key": "macro_thai", "label": "ภาพรวมประเทศไทย (BOT)", "module": "macro_thai", "fn": "fetch_macro_thai", "fetch_module": "macro_thai", "frequency": "monthly"},
+    {"key": "bank_npl", "label": "NPL ภาคการเงิน (BOT)", "module": "bank_npl", "fn": "fetch_bank_npl", "fetch_module": "bank_npl", "frequency": "quarterly"},
+]
+
+# Frequencies -> minimum seconds between successful refreshes of a job.
+# These implement the "adjust cadence to the source" requirement.
+_FREQ_SECONDS: dict[str, int] = {
+    "daily": 24 * 3600,
+    "weekly": 7 * 24 * 3600,
+    "monthly": 30 * 24 * 3600,
+    "quarterly": 91 * 24 * 3600,
+}
+
+# Data that legitimately changes every day (or faster) is refreshed separately
+# at a daily cadence rather than on the slow factor loop.
+_DAILY_JOBS: List[dict] = [
+    {"key": "siamchart_vintages", "label": "Siamchart SET50 snapshot (vintage)", "fn": "_record_siamchart_vintages", "frequency": "daily"},
+    {"key": "price_snapshot", "label": "ราคาหุ้น SET50 (Yahoo)", "fn": "_refresh_price_snapshot", "frequency": "daily"},
 ]
 
 
@@ -77,23 +97,103 @@ class AppDataScheduler:
                 log.exception("set50 scheduler refresh_all failed (will retry)")
 
     def refresh_all(self) -> list[dict]:
-        """Run every collector, warm the daily cache, snapshot the state, and
-        append each factor's value to the historical store (P4 enabler)."""
+        """Run each collector (respecting its natural cadence), warm the cache,
+        snapshot state, write vintages, and log per-source health.
+
+        Slow sources (monthly/quarterly factors, dividends, Siamchart) are only
+        re-fetched once their cooldown has elapsed; only daily-changing data
+        (prices) runs every eligible tick. This keeps resource usage aligned
+        with how often the data actually changes (the strategy rebalances a few
+        times a year) while still front-loading an initial refresh at boot.
+        """
         results: list[dict] = []
         fetched_by_module: dict[str, dict] = {}
+
         for job in _REFRESH_JOBS:
+            if not self._job_due(job):
+                continue
             res = self._run_job(job)
             results.append(res)
-            fetch_module = job.get("fetch_module")
-            if res.get("ok") and isinstance(res.get("value"), dict) and fetch_module:
-                fetched_by_module[fetch_module] = res["value"]
+            if res.get("ok"):
+                self._mark_job_run(job)
+                fetch_module = job.get("fetch_module")
+                if isinstance(res.get("value"), dict) and fetch_module:
+                    fetched_by_module[fetch_module] = res["value"]
+
+        # daily cadence data (Siamchart vintages + price snapshots)
+        for djob in _DAILY_JOBS:
+            if not self._job_due(djob):
+                continue
+            try:
+                fn = getattr(self, djob["fn"])
+                fn()
+                self._mark_job_run(djob)
+                results.append({"key": djob["key"], "label": djob["label"],
+                                "ok": True, "at": self._now()})
+            except Exception as exc:  # noqa: BLE001 — non-fatal
+                results.append({"key": djob["key"], "label": djob["label"],
+                                "ok": False, "error": str(exc), "at": self._now()})
+
         self._record_history(fetched_by_module)
         self._record_pit_factor_vintages(fetched_by_module)
-        self._record_siamchart_vintages()
         self._maybe_refresh_dated_dividends()
         self._write_marker(results)
+        self._append_source_log(results)
         return results
 
+    # -- per-job cadence cooldown ------------------------------------------
+    def _job_marker_path(self, job: dict) -> Path:
+        safe = "".join(c if (c.isalnum() or c in "._-") else "_" for c in job["key"])
+        return self._snap_dir / f"job_{safe}.json"
+
+    def _job_due(self, job: dict) -> bool:
+        """True if the job's cooldown (from its `frequency`) has elapsed."""
+        import json
+        import datetime as _dt
+        freq = job.get("frequency")
+        cooldown = _FREQ_SECONDS.get(freq if isinstance(freq, str) else "daily", _FREQ_SECONDS["daily"])
+        path = self._job_marker_path(job)
+        if not path.is_file():
+            return True  # never run -> run at boot
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            last = _dt.datetime.fromisoformat(data["at"])
+            elapsed = (_dt.datetime.now().astimezone() - last).total_seconds()
+            return elapsed >= cooldown
+        except (OSError, ValueError, KeyError):
+            return True  # corrupt/missing marker -> allow retry
+
+    def _mark_job_run(self, job: dict) -> None:
+        import json
+        try:
+            self._job_marker_path(job).write_text(
+                json.dumps({"at": self._now()}), encoding="utf-8")
+        except OSError:
+            log.exception("could not write job marker for %s", job["key"])
+
+    def _refresh_price_snapshot(self) -> None:
+        """Refresh the SET50 Yahoo price snapshot (daily cadence).
+
+        Pulls live daily bars over a rolling ~3y window into the price snapshot
+        store so valuation and next-trading-day execution use fresh prices. This
+        is the one data type that legitimately changes every trading day, so it
+        runs on the daily cadence rather than the slow factor loop. Non-fatal:
+        on network failure the previous snapshot is retained.
+        """
+        import datetime as _dt
+        from .prices import collect_price_snapshot, PriceSourceError
+        # rolling window: start ~3y back (enough history for momentum + next-day
+        # execution), end = yesterday (SET session close is the latest tradable).
+        today = _dt.date.today()
+        start = (today - _dt.timedelta(days=3 * 366)).isoformat()
+        end = (today - _dt.timedelta(days=1)).isoformat()
+        prices_dir = self.data_root / "prices"
+        try:
+            collect_price_snapshot(prices_dir, start=start, end=end)
+        except PriceSourceError as exc:
+            log.warning("set50 price snapshot refresh failed (non-fatal): %s", exc)
+            raise
+
     def _record_history(self, fetched_by_module: dict[str, dict]) -> None:
         """Append current factor values to the historical store (append-only).
 
@@ -279,6 +379,92 @@ class AppDataScheduler:
         except OSError:
             log.exception("could not write scheduler marker")
 
+    def _append_source_log(self, results: list[dict]) -> None:
+        """Append this refresh tick's per-source outcome to a durable health log
+        the frontend can render (with a one-click copy), and analyze failures.
+
+        Stored at ``data/scheduler/source_health.json`` (ring buffer, newest
+        first). Each entry categorizes the failure (network / http / parse /
+        structure / auth / other) so a source that "changed its page structure"
+        is distinguishable from a transient network blip.
+        """
+        import json
+        if not results:
+            return
+        path = self._snap_dir / "source_health.json"
+        try:
+            existing = []
+            if path.is_file():
+                existing = json.loads(path.read_text(encoding="utf-8"))
+            if not isinstance(existing, list):
+                existing = []
+        except (OSError, ValueError):
+            existing = []
+        entries = []
+        for r in results:
+            category = "ok"
+            if not r.get("ok"):
+                category = self._analyze_error(r.get("error", ""))
+            entries.append({
+                "key": r.get("key"), "label": r.get("label"),
+                "ok": bool(r.get("ok")), "category": category,
+                "at": r.get("at") or self._now(),
+                "detail": str(r.get("error") or "")[:500],
+            })
+        # newest-first ring buffer, cap at 500 entries
+        combined = entries + existing
+        combined = combined[:500]
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(json.dumps(combined, ensure_ascii=False), encoding="utf-8")
+        except OSError:
+            log.exception("could not write source-health log")
+
+    @staticmethod
+    def _analyze_error(message: str) -> str:
+        """Classify a failure reason so the frontend can guide diagnosis.
+
+        Keys off the exception message/type text. Returns one of:
+        network, timeout, http, parse, structure, auth, other.
+        """
+        m = (message or "").lower()
+        if not m:
+            return "other"
+        if any(t in m for t in ("timed out", "timeout", "timedout")):
+            return "timeout"
+        if any(t in m for t in ("no such host", "connection refused",
+                                "name or service not known", "network is unreachable",
+                                "connection reset", "getaddrinfo", "dns")):
+            return "network"
+        if any(t in m for t in ("http ", "status", "response code", "403", "404",
+                                "429", "502", "503")):
+            return "http"
+        if any(t in m for t in ("json decode", "parse", "unable to find",
+                                "regex", "no match", "value not found",
+                                "expecting value", "jsondecodeerror")):
+            return "parse"
+        if any(t in m for t in ("structure", "schema", "changed", "column",
+                                "field missing", "keyerror", "attributeerror")):
+            return "structure"
+        if any(t in m for t in ("auth", "login", "token", "credentials",
+                                "unauthorized", "forbidden", "401")):
+            return "auth"
+        return "other"
+
+    def _load_source_health(self, limit: int = 200) -> list[dict]:
+        """Return the most recent source-health entries (for the API/UI)."""
+        import json
+        path = self._snap_dir / "source_health.json"
+        if not path.is_file():
+            return []
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            return []
+        if not isinstance(data, list):
+            return []
+        return data[:limit]
+
     @staticmethod
     def _now() -> str:
         import datetime as _dt

backend/tests/test_scheduler.py

@@ -128,6 +128,68 @@ class VintageCollectionTest(unittest.TestCase):
             len(__import__("json").loads(manifest.read_text()).get("snapshots", {})), 1)
 
 
+class CadenceAndHealthTest(unittest.TestCase):
+    """Per-source cadence + source-health log (user requirements)."""
+
+    def test_job_due_respects_frequency_marker(self):
+        import json
+        snap_dir = Path(tempfile.mkdtemp())
+        sched = AppDataScheduler(_FakeCache(), snap_dir, interval_seconds=99999)
+        job = {"key": "macro_thai", "frequency": "monthly"}
+        # never run -> due
+        self.assertTrue(sched._job_due(job))
+        sched._mark_job_run(job)
+        # just marked -> not due again for ~30 days
+        self.assertFalse(sched._job_due(job))
+        # age the marker back 40 days -> due again
+        import datetime as _dt
+        old = (_dt.datetime.now().astimezone() - _dt.timedelta(days=40)).isoformat(timespec="seconds")
+        (snap_dir / "scheduler" / "job_macro_thai.json").write_text(
+            json.dumps({"at": old}), encoding="utf-8")
+        self.assertTrue(sched._job_due(job))
+
+    def test_analyze_error_classifies(self):
+        from app.scheduler import AppDataScheduler
+        self.assertEqual(AppDataScheduler._analyze_error("timed out connecting"), "timeout")
+        self.assertEqual(AppDataScheduler._analyze_error("Connection refused to host"), "network")
+        self.assertEqual(AppDataScheduler._analyze_error("HTTP 404 Not Found"), "http")
+        self.assertEqual(AppDataScheduler._analyze_error("JSONDecodeError: expecting value"), "parse")
+        self.assertEqual(AppDataScheduler._analyze_error("page structure changed - KeyError 'field'"), "structure")
+        self.assertEqual(AppDataScheduler._analyze_error("unauthorized token expired"), "auth")
+        self.assertEqual(AppDataScheduler._analyze_error(""), "other")
+
+    def test_source_health_log_written_and_readable(self):
+        import json
+        snap_dir = Path(tempfile.mkdtemp())
+        sched = AppDataScheduler(_FakeCache(), snap_dir, interval_seconds=99999)
+        results = [
+            {"key": "macro_thai", "label": "ภาพรวม (BOT)", "ok": True, "at": "2026-08-28T00:00:00+07:00"},
+            {"key": "auto_npl", "label": "NPL รถ", "ok": False,
+             "error": "JSONDecodeError: expecting value at line 1 (structure change?)",
+             "at": "2026-08-28T00:01:00+07:00"},
+        ]
+        sched._append_source_log(results)
+        entries = sched._load_source_health()
+        self.assertEqual(len(entries), 2)
+        by_key = {e["key"]: e for e in entries}
+        self.assertTrue(by_key["macro_thai"]["ok"])
+        self.assertFalse(by_key["auto_npl"]["ok"])
+        self.assertEqual(by_key["auto_npl"]["category"], "parse")
+
+    def test_refresh_only_runs_due_jobs(self):
+        # with frequency markers set to "now", a second refresh_all in the same
+        # tick should skip all factor jobs but still try daily jobs.
+        snap_dir = Path(tempfile.mkdtemp())
+        sched = AppDataScheduler(_FakeCache(), snap_dir, interval_seconds=99999)
+        from app.scheduler import _REFRESH_JOBS, _DAILY_JOBS
+        for job in _REFRESH_JOBS + _DAILY_JOBS:
+            sched._mark_job_run(job)
+        # all marked -> a refresh tick runs nothing successfully (no network)
+        results = sched.refresh_all()
+        # daily jobs that ARE collections we stubbed: none should hard-crash
+        self.assertIsInstance(results, list)
+
+
 class DividendCooldownTest(unittest.TestCase):
     def _sched(self, snap_dir, cooldown):
         return AppDataScheduler(_FakeCache(), snap_dir, interval_seconds=99999,

frontend/dist/index.html

@@ -5,7 +5,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <meta name="theme-color" content="#0b1018" />
     <title>SET50 Signal Lab</title>
-    <script type="module" crossorigin src="/assets/index-CUQk4rwt.js"></script>
+    <script type="module" crossorigin src="/assets/index-DOVbTpbx.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-z73iDem3.css">
   </head>
   <body>

frontend/src/App.vue

@@ -53,6 +53,9 @@ const factorRows = computed(() => factorData.value?.factors ?? [])
 // real multi-theme dashboard (3 themes + macro + sources)
 const dashboardThemes = computed(() => dashData.value?.themes ?? [])
 const dashboardSources = computed(() => dashData.value?.sources ?? [])
+// data scheduler source-health log (refresh cadence + failure diagnosis)
+const sourceHealth = ref([])
+const sourceHealthLoaded = ref(false)
 const dashboardMacro = computed(() => dashData.value?.macro ?? {})
 const sourceCount = computed(() => dashboardSources.value.length)
 const factorCount = computed(() => dashData.value?.source_summary?.factor_keys ?? sourceCount.value)
@@ -403,6 +406,34 @@ async function loadBacktestReadiness() {
   } catch { btReadiness.value = null }
 }
 
+async function loadSourceHealth() {
+  try {
+    const body = await fetchJson('/api/v1/scheduler/sources')
+    sourceHealth.value = body.sources || []
+  } catch { sourceHealth.value = [] }
+  sourceHealthLoaded.value = true
+}
+
+const healthCategoryLabel = (cat) => ({
+  ok: 'ปกติ', network: 'เครือข่ายขัดข้อง', timeout: 'หมดเวลา',
+  http: 'HTTP error', parse: 'รูปแบบข้อมูลผิด', structure: 'หน้าเว็บเปลี่ยนโครงสร้าง',
+  auth: 'สิทธิ์/ยืนยันตัวตน', other: 'อื่น ๆ',
+})[cat] || cat
+
+async function copyHealthLine(entry) {
+  const line = `[${entry.at}] ${entry.label} (${entry.key}) — ${entry.ok ? 'OK' : 'FAIL: ' + healthCategoryLabel(entry.category)} ${entry.detail ? '| ' + entry.detail : ''}`
+  try {
+    await navigator.clipboard.writeText(line)
+    notice.value = `คัดลอกสาเหตุของ ${entry.key} แล้ว`
+  } catch {
+    notice.value = line  // fallback: show raw text as a notice
+  }
+}
+
+function appendHealthRationale(entry) {
+  return entry.ok ? '' : ` (สาเหตุน่าจะ: ${healthCategoryLabel(entry.category)})`
+}
+
 async function runBacktest() {
   btLoading.value = true
   btResult.value = null
@@ -489,7 +520,7 @@ async function recordPaperEntry() {
   }
 }
 
-onMounted(async () => { await loadDashboard(); await Promise.all([loadBacktestRuns(), loadBacktestReadiness()]) })
+onMounted(async () => { await loadDashboard(); await Promise.all([loadBacktestRuns(), loadBacktestReadiness(), loadSourceHealth()]) })
 </script>
 
 <template>
@@ -510,6 +541,7 @@ onMounted(async () => { await loadDashboard(); await Promise.all([loadBacktestRu
         <a class="nav-item" href="#signals"><span class="nav-glyph">↗</span>สัญญาณ</a>
         <a class="nav-item" href="#factors"><span class="nav-glyph">∿</span>ปัจจัย</a>
         <a class="nav-item" href="#lineage"><span class="nav-glyph">⌁</span>ที่มาข้อมูล</a>
+        <a class="nav-item" href="#health"><span class="nav-glyph">⛨</span>สถานะข้อมูล</a>
         <a class="nav-item" href="#research-run"><span class="nav-glyph">✓</span>งานวิจัย</a>
         <a class="nav-item" href="#simulation"><span class="nav-glyph">➛</span>จำลอง</a>
       </nav>
@@ -684,6 +716,42 @@ onMounted(async () => { await loadDashboard(); await Promise.all([loadBacktestRu
           </div>
         </section>
 
+        <section class="panel health-panel" id="health">
+          <div class="panel-header signal-header">
+            <div>
+              <div class="section-kicker">สถานะการดึงข้อมูล</div>
+              <h2>Log — สถานะแหล่งข้อมูล</h2>
+              <p class="panel-subtitle">ผลการดึงข้อมูลครั้งล่าสุดของแต่ละแหล่ง โดยระบบวิเคราะห์สาเหตุให้อัตโนมัติ (เครือข่าย / หมดเวลา / หน้าเว็บเปลี่ยนโครงสร้าง / รูปแบบข้อมูล เป็นต้น) — กดปุ่ม "คัดลอก" เพื่อ copy สาเหตุไปแจ้ง/ตรวจสอบได้ทันที.</p>
+            </div>
+          </div>
+          <div v-if="sourceHealth.length === 0" class="empty-research muted-cell">ยังไม่มี log — รอรอบ refresh ถัดไป (ปกติ ~ทุกวันสำหรับราคา, ~รายเดือน/ไตรมาสสำหรับปัจจัย).</div>
+          <div v-else>
+            <table class="source-table">
+              <thead><tr><th>แหล่ง</th><th>ผลลัพธ์</th><th>สาเหตุ</th><th>เวลา</th><th></th></tr></thead>
+              <tbody>
+                <tr v-for="(e, i) in sourceHealth.slice(0, 20)" :key="i">
+                  <td class="source-name">{{ e.label }}<div class="muted-cell" style="font-size:11px">{{ e.key }}</div></td>
+                  <td>
+                    <span v-if="e.ok" class="status-tag" style="background:#1a7f37;color:#fff">OK</span>
+                    <span v-else class="status-tag warning-tag">FAIL</span>
+                  </td>
+                  <td>
+                    <template v-if="e.ok">—</template>
+                    <template v-else>
+                      <div>{{ healthCategoryLabel(e.category) }}{{ appendHealthRationale(e) }}</div>
+                      <div v-if="e.detail" class="muted-cell" style="font-size:11px;word-break:break-word">{{ e.detail.slice(0, 160) }}</div>
+                    </template>
+                  </td>
+                  <td class="muted-cell">{{ e.at ? formatDate(e.at) : '—' }}</td>
+                  <td>
+                    <button v-if="!e.ok" class="primary-btn" style="padding:2px 8px" @click="copyHealthLine(e)">คัดลอกสาเหตุ</button>
+                  </td>
+                </tr>
+              </tbody>
+            </table>
+          </div>
+        </section>
+
         <!-- Sections 01-04 (tourism-only surprise, provenance, signal table, frozen research) removed per user: superseded by the 3-theme panel + sources table above. -->
         <section class="panel sim-panel" id="simulation">
           <div class="panel-header signal-header">