Files

325 lines
19 KiB
Python

"""MT5-only prices; unchanged PrimeThai rules; comparisons on identical dates."""
from collections import defaultdict
from datetime import datetime, timedelta, timezone
import csv
import hashlib
import io
import json
import math
from pathlib import Path
import re
import time
import zipfile
import numpy as np
import pandas as pd
from primethai.config import TradingCosts
from primethai.model_lab import (
_features, build_model_targets, build_union_target_weights,
build_next_session_union_signals, load_market_panel,
simulate_board_lots, calculate_model_metrics,
)
from .config import REPO, TZ
from .store import now_iso
MODELS = ("donchian_55_20", "ema50_200_cross")
EOD = re.compile(r"set-history_EOD_(\d{4}-\d{2}-\d{2})\.csv$", re.I)
def plain(value):
if isinstance(value, dict):
return {str(k): plain(v) for k, v in value.items()}
if isinstance(value, (list, tuple)):
return [plain(v) for v in value]
if isinstance(value, (datetime, pd.Timestamp)):
return value.isoformat()
if hasattr(value, 'item'):
return plain(value.item())
if isinstance(value, float) and not math.isfinite(value):
return None
return value
def download_history(broker, store, settings, progress, *, force=False):
through = settings.closed_through()
benchmark_ready = store.last_bar_day('SET', through) == through
if store.get('history_checked_through') == through and benchmark_ready and not force:
progress('history', 'ใช้ประวัติ MT5 ที่เก็บไว้แล้ว', 1, 1)
return store.get('history_download', {})
symbols = [s.name for s in broker.symbols()
if 'Common Stock' in s.path and s.path.split('\\')[0] in ('SET', 'MAI')
and re.fullmatch(r'[A-Z0-9]{1,5}', s.name)]
symbols = sorted(set(symbols) | {'SET'})
previous_record = store.get('history_download', {})
previous = set(previous_record.get('successful_symbols', []))
successful, failures, zero_volume = [], [], []
for number, symbol in enumerate(symbols, 1):
progress('history', f'อ่าน D1 {symbol} · {number}/{len(symbols)}', number, len(symbols))
try:
symbol_state = store.get('history_symbol:' + symbol, {})
cached = (symbol_state.get('through') == through
or (symbol in previous and previous_record.get('through') == through))
# A successful download request is not proof the benchmark reached
# the requested session. Retry SET until the actual D1 bar arrives.
if cached and not force and (symbol != 'SET' or benchmark_ready):
successful.append(symbol)
if symbol in previous_record.get('zero_real_volume', []):
zero_volume.append(symbol)
continue
prior_day = symbol_state.get('through', previous_record.get('through'))
gap = (datetime.fromisoformat(through) - datetime.fromisoformat(prior_day)).days if prior_day else 999
count = settings.history_bars if force or symbol not in previous or gap > 20 else 30
rates = broker.history(symbol, count)
rows = []
for r in rates:
# D1 calendar dates label broker sessions; keep that raw date.
# Intraday quote/server-clock conversion must not shift D1 labels.
day = datetime.fromtimestamp(int(r['time']), timezone.utc).date().isoformat()
prices = [float(r[c]) for c in ('open', 'high', 'low', 'close')]
if day < settings.warmup_start or day > through:
continue
if not all(math.isfinite(p) and p > 0 for p in prices):
continue
rows.append((symbol, day, *prices, float(r['real_volume']), float(r['tick_volume'])))
if not rows:
raise ValueError('ไม่มีแท่งปิดในช่วงที่ต้องใช้')
if symbol != 'SET' and all(r[6] <= 0 for r in rows):
zero_volume.append(symbol)
store.save_bars(rows)
latest_day = max(r[1] for r in rows)
store.put('history_symbol:' + symbol, {'through': through, 'latest_day': latest_day,
'rows': len(rows), 'checked_at': now_iso()})
if symbol == 'SET' and latest_day != through:
raise ValueError(f'SET D1 ล่าสุด {latest_day}; ต้องถึง {through} จะลองดึงใหม่ในรอบถัดไป')
successful.append(symbol)
except Exception as exc:
failures.append({'symbol': symbol, 'error': str(exc)})
result = {'checked_at': now_iso(), 'through': through, 'requested': len(symbols),
'successful_symbols': successful, 'failures': failures, 'zero_real_volume': zero_volume,
'bar_summary': store.bar_summary()}
store.put('history_download', result)
if 'SET' not in successful:
raise ValueError('ไม่มีข้อมูล SET จาก MT5 จึงยังคำนวณ market filter ไม่ได้')
# Retry incomplete downloads on the next scheduled pass.
if not failures:
store.put('history_checked_through', through)
return result
def export_archive(store, settings):
path = settings.state_dir / 'mt5_eod.zip'
rows = store.bars(settings.warmup_start, settings.closed_through())
by_day = defaultdict(list)
for r in rows:
by_day[r['day']].append(r)
if not by_day:
raise ValueError('ยังไม่มีประวัติราคาที่ปิดแล้ว')
# Some individual MT5 symbols contain bars on market holidays. A session
# without a SET bar must not enter the common calendar: it would introduce
# a missing benchmark observation and invalidate its 200-session average.
sessions = {r['day'] for r in rows if r['symbol'] == 'SET'}
if not sessions or max(by_day) > max(sessions):
raise ValueError('ดัชนี SET ยังไม่ถึงวันล่าสุดของหุ้น รอข้อมูลก่อนคำนวณสัญญาณ')
excluded = sorted(set(by_day) - sessions)
quality = {'calendar_source': 'MT5 SET D1',
'excluded_non_session_days': [
{'day': day, 'rows': len(by_day[day]),
'symbols': [r['symbol'] for r in by_day[day]]} for day in excluded],
'raw_rows_retained': True}
store.put('data_quality', quality)
temporary = path.with_suffix('.tmp')
with zipfile.ZipFile(temporary, 'w', compression=zipfile.ZIP_DEFLATED) as archive:
for day, records in sorted(by_day.items()):
if day not in sessions:
continue
out = io.StringIO(newline='')
writer = csv.writer(out)
writer.writerow(['<TICKER>', '<DTYYYYMMDD>', '<OPEN>', '<HIGH>', '<LOW>', '<CLOSE>', '<VOL>'])
for r in records:
writer.writerow([r['symbol'], day.replace('-', ''), r['open'], r['high'], r['low'], r['close'], r['volume']])
archive.writestr(f'set-history_EOD_{day}.csv', out.getvalue())
temporary.replace(path)
return path, max(sessions)
def legacy_archives():
base = REPO / 'data/raw/siamchart'
return [p for p in (base / 'set-archive_EOD_1970-LAST.zip', base / 'set-archive_EOD.zip',
base / 'set-archive_EOD_LAST.zip') if p.is_file()]
def latest_archive_date(paths):
dates = []
for path in paths:
with zipfile.ZipFile(path) as archive:
dates.extend(m.group(1) for name in archive.namelist() if (m := EOD.search(Path(name).name)))
return max(dates) if dates else None
def panel_for(paths, settings, through):
end = (datetime.fromisoformat(through) + timedelta(days=1)).date().isoformat()
panel = load_market_panel(
paths, start=settings.warmup_start, end=end,
action_dir=REPO / 'data/raw/actions',
set_index_csv=settings.state_dir / 'NO_EXTERNAL_SET_FALLBACK.csv',
set_reference=REPO / 'data/reference/set_actions_2025_2026_h1.csv',
)
if len(panel.dates) < 253 or not math.isfinite(float(panel.set_close.iloc[-1])):
raise ValueError('ประวัติ warm-up ไม่พอ หรือไม่มีดัชนี SET ในวันล่าสุด จึงไม่ออกสัญญาณใหม่')
return panel
def calculate(panel, settings):
features = _features(panel)
signals = build_next_session_union_signals(panel, features, start=settings.model_start,
initial_equity=10_000., board_lot=100)
targets = [build_model_targets(panel, features, model=m, start=settings.model_start) for m in MODELS]
return features, plain(signals.to_dict('records')), build_union_target_weights(*targets)
def backtest(panel, targets, settings):
curve, execution = simulate_board_lots(panel, targets.gt(0), target_weights=targets,
initial_equity=10_000., board_lot=100, max_positions=20,
costs=TradingCosts())
end = (panel.dates[-1] + pd.Timedelta(days=1)).date().isoformat()
metrics, _ = calculate_model_metrics(curve, start=settings.model_start, end=end, execution=execution)
cropped = curve.loc[curve.index >= settings.model_start]
metrics['ending_equity'] = float(cropped['equity'].iloc[-1])
return plain({'metrics': metrics,
'curve': [{'day': str(day.date()), 'equity': float(r.equity)} for day, r in cropped.iterrows()],
'initial_equity': 10_000, 'costs': {'commission_rate': .0015, 'slippage_bps': 5}})
def compare_panels(left, right, left_signals, right_signals, left_features, right_features):
days = left.dates.intersection(right.dates)
symbols = sorted(set(left.symbols) & set(right.symbols))
missing_left = sorted(set(right.symbols) - set(left.symbols))
missing_right = sorted(set(left.symbols) - set(right.symbols))
fields, samples = {}, []
for field in ('open', 'high', 'low', 'close', 'volume'):
a = getattr(left, field).reindex(index=days, columns=symbols).to_numpy()
b = getattr(right, field).reindex(index=days, columns=symbols).to_numpy()
mask = (left.has_bar.reindex(index=days, columns=symbols).to_numpy()
& right.has_bar.reindex(index=days, columns=symbols).to_numpy())
matches = np.isclose(a, b, atol=1e-6, rtol=1e-8) & mask
differences = mask & ~matches
fields[field] = {'compared': int(mask.sum()), 'matched': int(matches.sum()),
'different': int(differences.sum()),
'match_pct': float(matches.sum() / mask.sum() * 100) if mask.any() else None}
if field == 'close':
for i, j in list(zip(*np.where(differences)))[-80:]:
samples.append({'day': str(days[i].date()), 'symbol': symbols[j],
'mt5': float(a[i, j]), 'siamchart': float(b[i, j])})
ls = {s['symbol']: s for s in left_signals}
rs = {s['symbol']: s for s in right_signals}
lr = {r['symbol']: i + 1 for i, r in enumerate(left_signals)}
rr = {r['symbol']: i + 1 for i, r in enumerate(right_signals)}
signals = []
for s in sorted(set(ls) | set(rs)):
a, b = ls.get(s, {}), rs.get(s, {})
same = a.get('source_model', '') == b.get('source_model', '')
row = {'symbol': s, 'mt5': a.get('source_model', ''), 'siamchart': b.get('source_model', ''),
'same': same, 'mt5_rank': lr.get(s), 'siamchart_rank': rr.get(s),
'rank_same': lr.get(s) == rr.get(s)}
for field in ('target_weight', 'allocation_score', 'reference_close'):
row['mt5_' + field] = a.get(field)
row['siamchart_' + field] = b.get(field)
row['same_' + field] = (field in a and field in b and
math.isclose(float(a[field]), float(b[field]), abs_tol=1e-6, rel_tol=1e-8))
signals.append(row)
set_left = left.set_close.reindex(days).to_numpy()
set_right = right.set_close.reindex(days).to_numpy()
set_mask = np.isfinite(set_left) & np.isfinite(set_right)
same_signals = all(s['same'] for s in signals)
latest_common = str(days[-1].date()) if len(days) else None
universe_left = set(left_features['universe'].iloc[-1].loc[lambda v: v].index)
universe_right = set(right_features['universe'].iloc[-1].loc[lambda v: v].index)
left_mask = left.has_bar.reindex(index=days, columns=symbols).to_numpy()
right_mask = right.has_bar.reindex(index=days, columns=symbols).to_numpy()
return plain({
'as_of': latest_common, 'same_signal_membership_and_model': same_signals,
'same_rankings': all(s['rank_same'] for s in signals),
'same_allocation_scores': all(s['same_allocation_score'] for s in signals),
'same_target_weights': all(s['same_target_weight'] for s in signals),
'shared_sessions': len(days), 'shared_symbols': len(symbols), 'ohlcv': fields,
'bars_only_mt5': int((left_mask & ~right_mask).sum()),
'bars_only_siamchart': int((right_mask & ~left_mask).sum()),
'sessions_only_mt5': [str(d.date()) for d in left.dates.difference(right.dates)],
'sessions_only_siamchart': [str(d.date()) for d in right.dates.difference(left.dates)],
'missing_from_mt5': missing_left, 'missing_from_siamchart': missing_right,
'set_compared': int(set_mask.sum()),
'set_different': int((set_mask & ~np.isclose(set_left, set_right, atol=1e-6, rtol=1e-8)).sum()),
'market_filter_mt5': {'close': float(left.set_close.iloc[-1]),
'sma200': float(left.set_close.rolling(200).mean().iloc[-1]),
'up': bool(left_features['market_up'].iloc[-1])},
'market_filter_siamchart': {'close': float(right.set_close.iloc[-1]),
'sma200': float(right.set_close.rolling(200).mean().iloc[-1]),
'up': bool(right_features['market_up'].iloc[-1])},
'signals': signals, 'mt5_count': len(ls), 'siamchart_count': len(rs),
'common_signal_count': len(set(ls) & set(rs)),
'latest_universe_mt5_only': sorted(universe_left - universe_right),
'latest_universe_siamchart_only': sorted(universe_right - universe_left),
'close_difference_samples': samples,
'notes': [
'ใช้กฎเดิม ช่วง warm-up เดียวกัน ข้อมูลปิดแล้ว และข้อมูลปันผล/แตกหุ้นชุดเดียวกัน',
'volume ใช้ real_volume เป็นจำนวนหุ้น ไม่ใช้ tick_volume แทน',
'MT5 แสดงหุ้นที่โบรกเกอร์มีในปัจจุบัน อาจไม่มีหุ้นที่เพิกถอนในประวัติเดิม',
'รายงานความต่างตามข้อมูลจริง ไม่มีการปรับราคา/volume ให้ดูตรงกัน',
],
})
def run_research(store, settings, progress):
progress('archive', 'เตรียมราคาจาก MT5 ที่เก็บในเครื่อง', 0, 1)
archive, through = export_archive(store, settings)
progress('signals', 'คำนวณ Donchian 55/20 และ EMA 50/200 จาก MT5', 0, 1)
current = panel_for([archive], settings, through)
features, signals, targets = calculate(current, settings)
output = {'updated_at': now_iso(), 'signal_as_of': str(current.dates[-1].date()),
'signals': signals, 'sessions': len(current.dates), 'symbols': len(current.symbols),
'history_issues': current.issues, 'warmup_start': settings.warmup_start,
'data_quality': store.get('data_quality', {}),
'model_start': settings.model_start,
'market_filter': {'set_close': float(current.set_close.iloc[-1]),
'sma200': float(current.set_close.rolling(200).mean().iloc[-1]),
'up': bool(features['market_up'].iloc[-1])},
'strategy_sha256': hashlib.sha256((REPO / 'primethai/model_lab.py').read_bytes()).hexdigest(),
'actions_note': 'ราคาจาก MT5; การปรับปันผล/แตกหุ้นใช้อ้างอิงที่มีอยู่ในโครงการเดิม',
'backtest_mt5_latest': backtest(current, targets, settings)}
archives = legacy_archives()
legacy_day = latest_archive_date(archives)
if legacy_day:
common_day = min(through, legacy_day)
progress('comparison', f'เทียบสองแหล่งข้อมูลถึงวันเดียวกัน {common_day}', 0, 1)
lp = panel_for([archive], settings, common_day)
rp = panel_for(archives, settings, common_day)
# Never label comparisons as equal-date if a whole market session is missing.
actual_common = min(lp.dates[-1], rp.dates[-1]).date().isoformat()
if str(lp.dates[-1].date()) != actual_common:
lp = panel_for([archive], settings, actual_common)
if str(rp.dates[-1].date()) != actual_common:
rp = panel_for(archives, settings, actual_common)
lf, ls, lt = calculate(lp, settings)
rf, rs, rt = calculate(rp, settings)
comparison = compare_panels(lp, rp, ls, rs, lf, rf)
comparison.update({'legacy_latest': legacy_day, 'mt5_latest': output['signal_as_of'],
'data_quality': output['data_quality'],
'backtest_mt5': backtest(lp, lt, settings),
'backtest_siamchart': backtest(rp, rt, settings)})
shared = lt.index.intersection(rt.index)
names = sorted(set(lt.columns) | set(rt.columns))
different_holdings = (lt.reindex(index=shared, columns=names, fill_value=0).gt(0)
!= rt.reindex(index=shared, columns=names, fill_value=0).gt(0))
comparison['historical_target_days_compared'] = len(shared[shared >= settings.model_start])
comparison['historical_target_days_different'] = int(different_holdings.loc[shared >= settings.model_start].any(axis=1).sum())
output['comparison'] = comparison
(settings.state_dir / 'comparison.json').write_text(json.dumps(plain(comparison), ensure_ascii=False, indent=2), encoding='utf-8')
pd.DataFrame(comparison['signals']).to_csv(settings.state_dir / 'signal_comparison.csv', index=False)
else:
output['comparison'] = {'error': 'ไม่พบไฟล์ SiamChart เดิมสำหรับเทียบ; สัญญาณหลักยังใช้ MT5'}
output = plain(output)
store.put('research', output)
(settings.state_dir / 'research.json').write_text(json.dumps(output, ensure_ascii=False, indent=2), encoding='utf-8')
return output