Files
set50-system/backend/tests/test_weight_learning.py
Kunthawat Greethong ae814c341e [verified] Factor-learning validation gate (no auto-apply)
Add a strict holdout/walk-forward + baseline gate to factor-weight learning,
per the P4 guardrail: learned weights are never auto-applied until minimum
sample, holdout/walk-forward, and baseline comparison all pass.

- backend/app/weight_learning.py:
  - FactorLearning gained ic_train / ic_holdout / validated / gate_notes.
  - apply_validation_gate(...) splits a chronological IC series into train +
    holdout and only marks validated=True when: total >= MIN_SAMPLE_PERIODS,
    each window >= its minimum, train AND holdout IC are positive (beat the
    BASELINE_IC=0) and agree in sign, and the pooled |t| > MIN_IC_TSTAT.
  - apply_weight_update now keeps new_weight == old_weight for any factor
    that is not validated (no auto-apply); only validated factors move.
  - learn_momentum_gated(...) builds PIT momentum ICs then applies the gate.
- backend/app/__init__.py: /api/v1/learning/momentum uses the gated learner
  and surfaces ic_train/ic_holdout/validated/gate_notes.
- tests: gate (16) via rewritten suite — full backend 286 passed.

Live probe on current price archive: validated=false with
gate_note 'IC not above baseline (0.0711/-0.1143)' — momentum is not
validated, weight stays unchanged (new_weight=None).
2026-08-27 12:17:21 +07:00

148 lines
5.9 KiB
Python

"""Tests for the factor-weight learning loop (P4) + validation gate."""
from __future__ import annotations
import datetime as dt
import unittest
from app import weight_learning as wl
class SpearmanICTest(unittest.TestCase):
def test_perfect_positive(self):
fv = {"A": 1.0, "B": 2.0, "C": 3.0, "D": 4.0}
fr = {"A": 0.1, "B": 0.2, "C": 0.3, "D": 0.4}
self.assertTrue(abs(wl.spearman_ic(fv, fr) - 1.0) < 1e-5)
def test_inverse_is_negative_one(self):
fv = {"A": 1.0, "B": 2.0, "C": 3.0, "D": 4.0}
fr = {"A": 0.4, "B": 0.3, "C": 0.2, "D": 0.1}
self.assertTrue(abs(wl.spearman_ic(fv, fr) + 1.0) < 1e-5)
def test_too_few_symbols_returns_none(self):
self.assertIsNone(wl.spearman_ic({"A": 1.0}, {"A": 0.1}))
def test_invalid_symbols_excluded(self):
fv = {"A": 1.0, "B": 2.0, "C": 3.0, "D": 4.0, "E": float("nan")}
fr = {"A": 0.1, "B": 0.2, "C": 0.3, "D": 0.4, "E": 0.5}
self.assertTrue(abs(wl.spearman_ic(fv, fr) - 1.0) < 1e-5)
class ValidationGateTest(unittest.TestCase):
def test_passes_when_both_windows_positive_and_significant(self):
l = wl.learn_factor_series([0.2] * 10)
# train (10 positive, all strong) + holdout (6 positive) -> validated
train = [0.2] * 10
holdout = [0.3] * 6
res = wl.apply_validation_gate(l, train, holdout)
self.assertTrue(res.validated)
self.assertEqual(res.gate_notes, [])
self.assertIsNotNone(res.ic_train)
self.assertIsNotNone(res.ic_holdout)
def test_fails_when_sample_too_small(self):
l = wl.learn_factor_series([0.2] * 6)
res = wl.apply_validation_gate(l, [0.2] * 3, [0.3] * 3)
self.assertFalse(res.validated)
self.assertTrue(any("sample too small" in n for n in res.gate_notes))
def test_fails_when_holdout_not_above_baseline(self):
l = wl.learn_factor_series([0.2] * 10)
res = wl.apply_validation_gate(l, [0.2] * 10, [-0.1] * 6)
self.assertFalse(res.validated)
self.assertTrue(any("baseline" in n for n in res.gate_notes))
def test_fails_when_train_and_holdout_disagree_in_sign(self):
l = wl.learn_factor_series([0.2] * 10)
res = wl.apply_validation_gate(l, [0.2] * 10, [-0.05] * 6)
self.assertFalse(res.validated)
def test_fails_when_ic_not_significant(self):
l = wl.learn_factor_series([0.02] * 10)
res = wl.apply_validation_gate(l, [0.02] * 10, [0.02] * 6)
self.assertFalse(res.validated) # tiny IC, |t| below bar
class WeightUpdateGateTest(unittest.TestCase):
def test_unvalidated_factor_never_auto_applies(self):
# positive IC but NOT validated -> weight must NOT move
l = wl.FactorLearning("f", n_periods=20, ic_mean=0.3, old_weight=1.0)
wl.apply_weight_update(l, shrink=0.5)
self.assertEqual(l.new_weight, 1.0) # unchanged (no auto-apply)
self.assertFalse(l.validated)
def test_validated_positive_ic_raises_weight(self):
l = wl.FactorLearning("f", n_periods=20, ic_mean=0.3, old_weight=1.0,
validated=True)
wl.apply_weight_update(l, shrink=0.5)
self.assertIsNotNone(l.new_weight)
self.assertTrue(l.new_weight > 1.0)
def test_validated_negative_ic_lowers_weight(self):
l = wl.FactorLearning("f", n_periods=20, ic_mean=-0.4, old_weight=1.0,
validated=True)
wl.apply_weight_update(l, shrink=0.5)
self.assertIsNotNone(l.new_weight)
self.assertTrue(l.new_weight < 1.0)
def test_clamped_to_bounds(self):
l = wl.FactorLearning("f", n_periods=20, ic_mean=10.0, old_weight=1.0,
validated=True)
wl.apply_weight_update(l, shrink=1.0, max_w=3.0)
self.assertEqual(l.new_weight, 3.0)
def test_blocked_keeps_weight(self):
l = wl.FactorLearning("f", n_periods=0, ic_mean=None, old_weight=1.0,
blocked=True)
wl.apply_weight_update(l)
self.assertEqual(l.new_weight, 1.0)
def test_no_old_weight_returns(self):
l = wl.FactorLearning("f", n_periods=12, ic_mean=0.2, old_weight=None,
validated=True)
wl.apply_weight_update(l)
self.assertIsNone(l.new_weight)
class MomentumGatedLearningTest(unittest.TestCase):
@staticmethod
def _series():
def bars(base, drift):
out = []
for i in range(600):
d = (dt.date(2024, 1, 1) + dt.timedelta(days=i)).isoformat()
out.append({"date": d, "adjusted_close": base + drift * i})
return out
return {"A": {"bars": bars(10.0, 0.05)}, "B": {"bars": bars(20.0, 0.0)}}
def test_gated_runs_and_returns_validated_flag(self):
res = wl.learn_momentum_gated(self._series(), ["A", "B"],
"2025-01-01", "2026-06-01")
self.assertIsInstance(res, wl.FactorLearning)
self.assertIn(res.validated, (True, False)) # always a boolean verdict
self.assertGreaterEqual(res.n_periods, 0)
class AddMonthsTest(unittest.TestCase):
def test_clamps_day_to_end_of_month(self):
self.assertEqual(wl._add_months(dt.date(2026, 1, 31), 1), dt.date(2026, 2, 28))
self.assertEqual(wl._add_months(dt.date(2026, 1, 31), 2), dt.date(2026, 3, 31))
self.assertEqual(wl._add_months(dt.date(2024, 1, 31), 1), dt.date(2024, 2, 29))
class LearnFactorSeriesTest(unittest.TestCase):
def test_aggregates_ics(self):
res = wl.learn_factor_series([0.1, 0.2, 0.3, 0.4])
self.assertEqual(res.n_periods, 4)
self.assertTrue(res.ic_mean is not None and abs(res.ic_mean - 0.25) < 1e-6)
self.assertIsNotNone(res.ic_tstat)
def test_empty_is_blocked_like(self):
res = wl.learn_factor_series([])
self.assertEqual(res.n_periods, 0)
self.assertIsNone(res.ic_mean)
if __name__ == "__main__":
unittest.main()