Add a strict holdout/walk-forward + baseline gate to factor-weight learning,
per the P4 guardrail: learned weights are never auto-applied until minimum
sample, holdout/walk-forward, and baseline comparison all pass.
- backend/app/weight_learning.py:
- FactorLearning gained ic_train / ic_holdout / validated / gate_notes.
- apply_validation_gate(...) splits a chronological IC series into train +
holdout and only marks validated=True when: total >= MIN_SAMPLE_PERIODS,
each window >= its minimum, train AND holdout IC are positive (beat the
BASELINE_IC=0) and agree in sign, and the pooled |t| > MIN_IC_TSTAT.
- apply_weight_update now keeps new_weight == old_weight for any factor
that is not validated (no auto-apply); only validated factors move.
- learn_momentum_gated(...) builds PIT momentum ICs then applies the gate.
- backend/app/__init__.py: /api/v1/learning/momentum uses the gated learner
and surfaces ic_train/ic_holdout/validated/gate_notes.
- tests: gate (16) via rewritten suite — full backend 286 passed.
Live probe on current price archive: validated=false with
gate_note 'IC not above baseline (0.0711/-0.1143)' — momentum is not
validated, weight stays unchanged (new_weight=None).
148 lines
5.9 KiB
Python
148 lines
5.9 KiB
Python
"""Tests for the factor-weight learning loop (P4) + validation gate."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import unittest
|
|
|
|
from app import weight_learning as wl
|
|
|
|
|
|
class SpearmanICTest(unittest.TestCase):
|
|
def test_perfect_positive(self):
|
|
fv = {"A": 1.0, "B": 2.0, "C": 3.0, "D": 4.0}
|
|
fr = {"A": 0.1, "B": 0.2, "C": 0.3, "D": 0.4}
|
|
self.assertTrue(abs(wl.spearman_ic(fv, fr) - 1.0) < 1e-5)
|
|
|
|
def test_inverse_is_negative_one(self):
|
|
fv = {"A": 1.0, "B": 2.0, "C": 3.0, "D": 4.0}
|
|
fr = {"A": 0.4, "B": 0.3, "C": 0.2, "D": 0.1}
|
|
self.assertTrue(abs(wl.spearman_ic(fv, fr) + 1.0) < 1e-5)
|
|
|
|
def test_too_few_symbols_returns_none(self):
|
|
self.assertIsNone(wl.spearman_ic({"A": 1.0}, {"A": 0.1}))
|
|
|
|
def test_invalid_symbols_excluded(self):
|
|
fv = {"A": 1.0, "B": 2.0, "C": 3.0, "D": 4.0, "E": float("nan")}
|
|
fr = {"A": 0.1, "B": 0.2, "C": 0.3, "D": 0.4, "E": 0.5}
|
|
self.assertTrue(abs(wl.spearman_ic(fv, fr) - 1.0) < 1e-5)
|
|
|
|
|
|
class ValidationGateTest(unittest.TestCase):
|
|
def test_passes_when_both_windows_positive_and_significant(self):
|
|
l = wl.learn_factor_series([0.2] * 10)
|
|
# train (10 positive, all strong) + holdout (6 positive) -> validated
|
|
train = [0.2] * 10
|
|
holdout = [0.3] * 6
|
|
res = wl.apply_validation_gate(l, train, holdout)
|
|
self.assertTrue(res.validated)
|
|
self.assertEqual(res.gate_notes, [])
|
|
self.assertIsNotNone(res.ic_train)
|
|
self.assertIsNotNone(res.ic_holdout)
|
|
|
|
def test_fails_when_sample_too_small(self):
|
|
l = wl.learn_factor_series([0.2] * 6)
|
|
res = wl.apply_validation_gate(l, [0.2] * 3, [0.3] * 3)
|
|
self.assertFalse(res.validated)
|
|
self.assertTrue(any("sample too small" in n for n in res.gate_notes))
|
|
|
|
def test_fails_when_holdout_not_above_baseline(self):
|
|
l = wl.learn_factor_series([0.2] * 10)
|
|
res = wl.apply_validation_gate(l, [0.2] * 10, [-0.1] * 6)
|
|
self.assertFalse(res.validated)
|
|
self.assertTrue(any("baseline" in n for n in res.gate_notes))
|
|
|
|
def test_fails_when_train_and_holdout_disagree_in_sign(self):
|
|
l = wl.learn_factor_series([0.2] * 10)
|
|
res = wl.apply_validation_gate(l, [0.2] * 10, [-0.05] * 6)
|
|
self.assertFalse(res.validated)
|
|
|
|
def test_fails_when_ic_not_significant(self):
|
|
l = wl.learn_factor_series([0.02] * 10)
|
|
res = wl.apply_validation_gate(l, [0.02] * 10, [0.02] * 6)
|
|
self.assertFalse(res.validated) # tiny IC, |t| below bar
|
|
|
|
|
|
class WeightUpdateGateTest(unittest.TestCase):
|
|
def test_unvalidated_factor_never_auto_applies(self):
|
|
# positive IC but NOT validated -> weight must NOT move
|
|
l = wl.FactorLearning("f", n_periods=20, ic_mean=0.3, old_weight=1.0)
|
|
wl.apply_weight_update(l, shrink=0.5)
|
|
self.assertEqual(l.new_weight, 1.0) # unchanged (no auto-apply)
|
|
self.assertFalse(l.validated)
|
|
|
|
def test_validated_positive_ic_raises_weight(self):
|
|
l = wl.FactorLearning("f", n_periods=20, ic_mean=0.3, old_weight=1.0,
|
|
validated=True)
|
|
wl.apply_weight_update(l, shrink=0.5)
|
|
self.assertIsNotNone(l.new_weight)
|
|
self.assertTrue(l.new_weight > 1.0)
|
|
|
|
def test_validated_negative_ic_lowers_weight(self):
|
|
l = wl.FactorLearning("f", n_periods=20, ic_mean=-0.4, old_weight=1.0,
|
|
validated=True)
|
|
wl.apply_weight_update(l, shrink=0.5)
|
|
self.assertIsNotNone(l.new_weight)
|
|
self.assertTrue(l.new_weight < 1.0)
|
|
|
|
def test_clamped_to_bounds(self):
|
|
l = wl.FactorLearning("f", n_periods=20, ic_mean=10.0, old_weight=1.0,
|
|
validated=True)
|
|
wl.apply_weight_update(l, shrink=1.0, max_w=3.0)
|
|
self.assertEqual(l.new_weight, 3.0)
|
|
|
|
def test_blocked_keeps_weight(self):
|
|
l = wl.FactorLearning("f", n_periods=0, ic_mean=None, old_weight=1.0,
|
|
blocked=True)
|
|
wl.apply_weight_update(l)
|
|
self.assertEqual(l.new_weight, 1.0)
|
|
|
|
def test_no_old_weight_returns(self):
|
|
l = wl.FactorLearning("f", n_periods=12, ic_mean=0.2, old_weight=None,
|
|
validated=True)
|
|
wl.apply_weight_update(l)
|
|
self.assertIsNone(l.new_weight)
|
|
|
|
|
|
class MomentumGatedLearningTest(unittest.TestCase):
|
|
@staticmethod
|
|
def _series():
|
|
def bars(base, drift):
|
|
out = []
|
|
for i in range(600):
|
|
d = (dt.date(2024, 1, 1) + dt.timedelta(days=i)).isoformat()
|
|
out.append({"date": d, "adjusted_close": base + drift * i})
|
|
return out
|
|
return {"A": {"bars": bars(10.0, 0.05)}, "B": {"bars": bars(20.0, 0.0)}}
|
|
|
|
def test_gated_runs_and_returns_validated_flag(self):
|
|
res = wl.learn_momentum_gated(self._series(), ["A", "B"],
|
|
"2025-01-01", "2026-06-01")
|
|
self.assertIsInstance(res, wl.FactorLearning)
|
|
self.assertIn(res.validated, (True, False)) # always a boolean verdict
|
|
self.assertGreaterEqual(res.n_periods, 0)
|
|
|
|
|
|
class AddMonthsTest(unittest.TestCase):
|
|
def test_clamps_day_to_end_of_month(self):
|
|
self.assertEqual(wl._add_months(dt.date(2026, 1, 31), 1), dt.date(2026, 2, 28))
|
|
self.assertEqual(wl._add_months(dt.date(2026, 1, 31), 2), dt.date(2026, 3, 31))
|
|
self.assertEqual(wl._add_months(dt.date(2024, 1, 31), 1), dt.date(2024, 2, 29))
|
|
|
|
|
|
class LearnFactorSeriesTest(unittest.TestCase):
|
|
def test_aggregates_ics(self):
|
|
res = wl.learn_factor_series([0.1, 0.2, 0.3, 0.4])
|
|
self.assertEqual(res.n_periods, 4)
|
|
self.assertTrue(res.ic_mean is not None and abs(res.ic_mean - 0.25) < 1e-6)
|
|
self.assertIsNotNone(res.ic_tstat)
|
|
|
|
def test_empty_is_blocked_like(self):
|
|
res = wl.learn_factor_series([])
|
|
self.assertEqual(res.n_periods, 0)
|
|
self.assertIsNone(res.ic_mean)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|