d87a1ada39
A. Cross-theme comparability: - compute_theme_surprises now weight-normalizes by total |weight| (weighted average), so every theme surprise on same [-1,1] scale regardless of factor count/weight (retail 0.189->0.145; auto_credit 1.0->0.64). B. Historical factor store (enables learning macro/demographic factors): - New factor_history.py: append-only per-factor JSONL, dedupes unchanged values, rejects non-finite, records every FACTORS value each scheduler run. - scheduler.py: jobs carry fetch_module; refresh_all records factor history (non-fatal); added bank_npl job. - GET /api/v1/learning/factors?min_points= reports n_points/learnable per factor so users see when P4 learning unlocks (validated query parsing). - weight_learning: generic learn_factor_series() aggregator (momentum reuses). Independent review deleg_5dd358e3 passed=true (empty security/logic arrays); its two robustness suggestions applied (finite guard in record(), clean 400 on bad min_points). 234 tests pass; Vite build passes.
128 lines
5.1 KiB
Python
128 lines
5.1 KiB
Python
"""Append-only historical store for FACTORS registry values (P4 enabler).
|
|
|
|
Background: the dashboard shows each alternative factor's *current* value, but
|
|
factor-weight learning (P4) needs each factor's value *as seen at many past
|
|
points in time* so we can compute forward-return attribution (IC). Retroactive
|
|
reconstruction from a single snapshot is impossible, so this store starts
|
|
recording every factor value from now on, once per scheduled run.
|
|
|
|
Each factor gets its own JSONL file under
|
|
`backend/data/factor_history/<factor_key>.jsonl`, one JSON object per line:
|
|
{"ts": "2026-08-27T07:00:00+07:00", "value": 16.2, "as_of": "..."}
|
|
|
|
Only appends when the value actually changed since the last recorded point, so
|
|
the series stays compact and `series()` returns distinct observations (useful
|
|
for a proper time-series / IC analysis rather than N duplicate rows).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import os
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
from . import factors as factors_mod
|
|
|
|
_DEFAULT_DIR = Path(__file__).resolve().parent.parent / "data" / "factor_history"
|
|
|
|
|
|
class FactorHistoryError(ValueError):
|
|
pass
|
|
|
|
|
|
class FactorHistory:
|
|
"""Append-only time-series store for factor values (per-factor JSONL)."""
|
|
|
|
def __init__(self, history_dir: Path = _DEFAULT_DIR) -> None:
|
|
self.history_dir = Path(history_dir)
|
|
self.history_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
def _path(self, factor_key: str) -> Path:
|
|
safe = "".join(c if (c.isalnum() or c in "._-") else "_" for c in factor_key)
|
|
return self.history_dir / f"{safe}.jsonl"
|
|
|
|
def last_value(self, factor_key: str) -> Optional[float]:
|
|
"""Most recent recorded value, or None if the factor has no history yet."""
|
|
path = self._path(factor_key)
|
|
if not path.exists():
|
|
return None
|
|
try:
|
|
with path.open(encoding="utf-8") as fh:
|
|
last = None
|
|
for line in fh:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
rec = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
last = rec
|
|
return float(last["value"]) if last and last.get("value") is not None else None
|
|
except OSError:
|
|
return None
|
|
|
|
def record(self, factor_key: str, value: Optional[float],
|
|
ts: Optional[str] = None, as_of: str = "") -> bool:
|
|
"""Append a point only if it differs from the last recorded value.
|
|
Returns True if a point was written."""
|
|
if value is None:
|
|
return False
|
|
try:
|
|
value = float(value)
|
|
except (TypeError, ValueError):
|
|
return False
|
|
if not math.isfinite(value):
|
|
return False
|
|
last = self.last_value(factor_key)
|
|
if last is not None and abs(last - value) < 1e-12:
|
|
return False
|
|
ts = ts or datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
path = self._path(factor_key)
|
|
try:
|
|
with path.open("a", encoding="utf-8") as fh:
|
|
fh.write(json.dumps({"ts": ts, "value": value, "as_of": as_of},
|
|
ensure_ascii=False) + "\n")
|
|
except OSError as exc:
|
|
raise FactorHistoryError(f"cannot write factor history: {exc}") from exc
|
|
return True
|
|
|
|
def series(self, factor_key: str) -> list[dict]:
|
|
"""Full chronological series [{ts, value, as_of}] for a factor."""
|
|
path = self._path(factor_key)
|
|
out: list[dict] = []
|
|
if not path.exists():
|
|
return out
|
|
try:
|
|
with path.open(encoding="utf-8") as fh:
|
|
for line in fh:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
rec = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if isinstance(rec, dict) and "value" in rec:
|
|
out.append(rec)
|
|
except OSError as exc:
|
|
raise FactorHistoryError(f"cannot read factor history: {exc}") from exc
|
|
return out
|
|
|
|
def record_all(self, fetched: dict, ts: Optional[str] = None) -> dict[str, bool]:
|
|
"""Record the current value of every FACTORS entry from a `fetched`
|
|
dict (fetch-module -> collector dict). Returns {factor_key: wrote_bool}."""
|
|
writes: dict[str, bool] = {}
|
|
for fkey, fact in factors_mod.FACTORS.items():
|
|
if not fact.get("fetch"):
|
|
continue
|
|
val = factors_mod.factor_value(fact, fetched.get(fact.get("fetch")))
|
|
# best-effort as-of: pull the collector's own period/as_of if present
|
|
collector = fetched.get(fact.get("fetch")) or {}
|
|
as_of = str(collector.get("as_of") or collector.get("period") or "")
|
|
writes[fkey] = self.record(fkey, val, ts=ts, as_of=as_of)
|
|
return writes
|