325 lines
19 KiB
Python
325 lines
19 KiB
Python
"""MT5-only prices; unchanged PrimeThai rules; comparisons on identical dates."""
|
|
from collections import defaultdict
|
|
from datetime import datetime, timedelta, timezone
|
|
import csv
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import re
|
|
import time
|
|
import zipfile
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
from primethai.config import TradingCosts
|
|
from primethai.model_lab import (
|
|
_features, build_model_targets, build_union_target_weights,
|
|
build_next_session_union_signals, load_market_panel,
|
|
simulate_board_lots, calculate_model_metrics,
|
|
)
|
|
from .config import REPO, TZ
|
|
from .store import now_iso
|
|
|
|
MODELS = ("donchian_55_20", "ema50_200_cross")
|
|
EOD = re.compile(r"set-history_EOD_(\d{4}-\d{2}-\d{2})\.csv$", re.I)
|
|
|
|
|
|
def plain(value):
|
|
if isinstance(value, dict):
|
|
return {str(k): plain(v) for k, v in value.items()}
|
|
if isinstance(value, (list, tuple)):
|
|
return [plain(v) for v in value]
|
|
if isinstance(value, (datetime, pd.Timestamp)):
|
|
return value.isoformat()
|
|
if hasattr(value, 'item'):
|
|
return plain(value.item())
|
|
if isinstance(value, float) and not math.isfinite(value):
|
|
return None
|
|
return value
|
|
|
|
|
|
def download_history(broker, store, settings, progress, *, force=False):
|
|
through = settings.closed_through()
|
|
benchmark_ready = store.last_bar_day('SET', through) == through
|
|
if store.get('history_checked_through') == through and benchmark_ready and not force:
|
|
progress('history', 'ใช้ประวัติ MT5 ที่เก็บไว้แล้ว', 1, 1)
|
|
return store.get('history_download', {})
|
|
symbols = [s.name for s in broker.symbols()
|
|
if 'Common Stock' in s.path and s.path.split('\\')[0] in ('SET', 'MAI')
|
|
and re.fullmatch(r'[A-Z0-9]{1,5}', s.name)]
|
|
symbols = sorted(set(symbols) | {'SET'})
|
|
previous_record = store.get('history_download', {})
|
|
previous = set(previous_record.get('successful_symbols', []))
|
|
successful, failures, zero_volume = [], [], []
|
|
for number, symbol in enumerate(symbols, 1):
|
|
progress('history', f'อ่าน D1 {symbol} · {number}/{len(symbols)}', number, len(symbols))
|
|
try:
|
|
symbol_state = store.get('history_symbol:' + symbol, {})
|
|
cached = (symbol_state.get('through') == through
|
|
or (symbol in previous and previous_record.get('through') == through))
|
|
# A successful download request is not proof the benchmark reached
|
|
# the requested session. Retry SET until the actual D1 bar arrives.
|
|
if cached and not force and (symbol != 'SET' or benchmark_ready):
|
|
successful.append(symbol)
|
|
if symbol in previous_record.get('zero_real_volume', []):
|
|
zero_volume.append(symbol)
|
|
continue
|
|
prior_day = symbol_state.get('through', previous_record.get('through'))
|
|
gap = (datetime.fromisoformat(through) - datetime.fromisoformat(prior_day)).days if prior_day else 999
|
|
count = settings.history_bars if force or symbol not in previous or gap > 20 else 30
|
|
rates = broker.history(symbol, count)
|
|
rows = []
|
|
for r in rates:
|
|
# D1 calendar dates label broker sessions; keep that raw date.
|
|
# Intraday quote/server-clock conversion must not shift D1 labels.
|
|
day = datetime.fromtimestamp(int(r['time']), timezone.utc).date().isoformat()
|
|
prices = [float(r[c]) for c in ('open', 'high', 'low', 'close')]
|
|
if day < settings.warmup_start or day > through:
|
|
continue
|
|
if not all(math.isfinite(p) and p > 0 for p in prices):
|
|
continue
|
|
rows.append((symbol, day, *prices, float(r['real_volume']), float(r['tick_volume'])))
|
|
if not rows:
|
|
raise ValueError('ไม่มีแท่งปิดในช่วงที่ต้องใช้')
|
|
if symbol != 'SET' and all(r[6] <= 0 for r in rows):
|
|
zero_volume.append(symbol)
|
|
store.save_bars(rows)
|
|
latest_day = max(r[1] for r in rows)
|
|
store.put('history_symbol:' + symbol, {'through': through, 'latest_day': latest_day,
|
|
'rows': len(rows), 'checked_at': now_iso()})
|
|
if symbol == 'SET' and latest_day != through:
|
|
raise ValueError(f'SET D1 ล่าสุด {latest_day}; ต้องถึง {through} จะลองดึงใหม่ในรอบถัดไป')
|
|
successful.append(symbol)
|
|
except Exception as exc:
|
|
failures.append({'symbol': symbol, 'error': str(exc)})
|
|
result = {'checked_at': now_iso(), 'through': through, 'requested': len(symbols),
|
|
'successful_symbols': successful, 'failures': failures, 'zero_real_volume': zero_volume,
|
|
'bar_summary': store.bar_summary()}
|
|
store.put('history_download', result)
|
|
if 'SET' not in successful:
|
|
raise ValueError('ไม่มีข้อมูล SET จาก MT5 จึงยังคำนวณ market filter ไม่ได้')
|
|
# Retry incomplete downloads on the next scheduled pass.
|
|
if not failures:
|
|
store.put('history_checked_through', through)
|
|
return result
|
|
|
|
|
|
def export_archive(store, settings):
|
|
path = settings.state_dir / 'mt5_eod.zip'
|
|
rows = store.bars(settings.warmup_start, settings.closed_through())
|
|
by_day = defaultdict(list)
|
|
for r in rows:
|
|
by_day[r['day']].append(r)
|
|
if not by_day:
|
|
raise ValueError('ยังไม่มีประวัติราคาที่ปิดแล้ว')
|
|
# Some individual MT5 symbols contain bars on market holidays. A session
|
|
# without a SET bar must not enter the common calendar: it would introduce
|
|
# a missing benchmark observation and invalidate its 200-session average.
|
|
sessions = {r['day'] for r in rows if r['symbol'] == 'SET'}
|
|
if not sessions or max(by_day) > max(sessions):
|
|
raise ValueError('ดัชนี SET ยังไม่ถึงวันล่าสุดของหุ้น รอข้อมูลก่อนคำนวณสัญญาณ')
|
|
excluded = sorted(set(by_day) - sessions)
|
|
quality = {'calendar_source': 'MT5 SET D1',
|
|
'excluded_non_session_days': [
|
|
{'day': day, 'rows': len(by_day[day]),
|
|
'symbols': [r['symbol'] for r in by_day[day]]} for day in excluded],
|
|
'raw_rows_retained': True}
|
|
store.put('data_quality', quality)
|
|
temporary = path.with_suffix('.tmp')
|
|
with zipfile.ZipFile(temporary, 'w', compression=zipfile.ZIP_DEFLATED) as archive:
|
|
for day, records in sorted(by_day.items()):
|
|
if day not in sessions:
|
|
continue
|
|
out = io.StringIO(newline='')
|
|
writer = csv.writer(out)
|
|
writer.writerow(['<TICKER>', '<DTYYYYMMDD>', '<OPEN>', '<HIGH>', '<LOW>', '<CLOSE>', '<VOL>'])
|
|
for r in records:
|
|
writer.writerow([r['symbol'], day.replace('-', ''), r['open'], r['high'], r['low'], r['close'], r['volume']])
|
|
archive.writestr(f'set-history_EOD_{day}.csv', out.getvalue())
|
|
temporary.replace(path)
|
|
return path, max(sessions)
|
|
|
|
|
|
def legacy_archives():
|
|
base = REPO / 'data/raw/siamchart'
|
|
return [p for p in (base / 'set-archive_EOD_1970-LAST.zip', base / 'set-archive_EOD.zip',
|
|
base / 'set-archive_EOD_LAST.zip') if p.is_file()]
|
|
|
|
|
|
def latest_archive_date(paths):
|
|
dates = []
|
|
for path in paths:
|
|
with zipfile.ZipFile(path) as archive:
|
|
dates.extend(m.group(1) for name in archive.namelist() if (m := EOD.search(Path(name).name)))
|
|
return max(dates) if dates else None
|
|
|
|
|
|
def panel_for(paths, settings, through):
|
|
end = (datetime.fromisoformat(through) + timedelta(days=1)).date().isoformat()
|
|
panel = load_market_panel(
|
|
paths, start=settings.warmup_start, end=end,
|
|
action_dir=REPO / 'data/raw/actions',
|
|
set_index_csv=settings.state_dir / 'NO_EXTERNAL_SET_FALLBACK.csv',
|
|
set_reference=REPO / 'data/reference/set_actions_2025_2026_h1.csv',
|
|
)
|
|
if len(panel.dates) < 253 or not math.isfinite(float(panel.set_close.iloc[-1])):
|
|
raise ValueError('ประวัติ warm-up ไม่พอ หรือไม่มีดัชนี SET ในวันล่าสุด จึงไม่ออกสัญญาณใหม่')
|
|
return panel
|
|
|
|
|
|
def calculate(panel, settings):
|
|
features = _features(panel)
|
|
signals = build_next_session_union_signals(panel, features, start=settings.model_start,
|
|
initial_equity=10_000., board_lot=100)
|
|
targets = [build_model_targets(panel, features, model=m, start=settings.model_start) for m in MODELS]
|
|
return features, plain(signals.to_dict('records')), build_union_target_weights(*targets)
|
|
|
|
|
|
def backtest(panel, targets, settings):
|
|
curve, execution = simulate_board_lots(panel, targets.gt(0), target_weights=targets,
|
|
initial_equity=10_000., board_lot=100, max_positions=20,
|
|
costs=TradingCosts())
|
|
end = (panel.dates[-1] + pd.Timedelta(days=1)).date().isoformat()
|
|
metrics, _ = calculate_model_metrics(curve, start=settings.model_start, end=end, execution=execution)
|
|
cropped = curve.loc[curve.index >= settings.model_start]
|
|
metrics['ending_equity'] = float(cropped['equity'].iloc[-1])
|
|
return plain({'metrics': metrics,
|
|
'curve': [{'day': str(day.date()), 'equity': float(r.equity)} for day, r in cropped.iterrows()],
|
|
'initial_equity': 10_000, 'costs': {'commission_rate': .0015, 'slippage_bps': 5}})
|
|
|
|
|
|
def compare_panels(left, right, left_signals, right_signals, left_features, right_features):
|
|
days = left.dates.intersection(right.dates)
|
|
symbols = sorted(set(left.symbols) & set(right.symbols))
|
|
missing_left = sorted(set(right.symbols) - set(left.symbols))
|
|
missing_right = sorted(set(left.symbols) - set(right.symbols))
|
|
fields, samples = {}, []
|
|
for field in ('open', 'high', 'low', 'close', 'volume'):
|
|
a = getattr(left, field).reindex(index=days, columns=symbols).to_numpy()
|
|
b = getattr(right, field).reindex(index=days, columns=symbols).to_numpy()
|
|
mask = (left.has_bar.reindex(index=days, columns=symbols).to_numpy()
|
|
& right.has_bar.reindex(index=days, columns=symbols).to_numpy())
|
|
matches = np.isclose(a, b, atol=1e-6, rtol=1e-8) & mask
|
|
differences = mask & ~matches
|
|
fields[field] = {'compared': int(mask.sum()), 'matched': int(matches.sum()),
|
|
'different': int(differences.sum()),
|
|
'match_pct': float(matches.sum() / mask.sum() * 100) if mask.any() else None}
|
|
if field == 'close':
|
|
for i, j in list(zip(*np.where(differences)))[-80:]:
|
|
samples.append({'day': str(days[i].date()), 'symbol': symbols[j],
|
|
'mt5': float(a[i, j]), 'siamchart': float(b[i, j])})
|
|
ls = {s['symbol']: s for s in left_signals}
|
|
rs = {s['symbol']: s for s in right_signals}
|
|
lr = {r['symbol']: i + 1 for i, r in enumerate(left_signals)}
|
|
rr = {r['symbol']: i + 1 for i, r in enumerate(right_signals)}
|
|
signals = []
|
|
for s in sorted(set(ls) | set(rs)):
|
|
a, b = ls.get(s, {}), rs.get(s, {})
|
|
same = a.get('source_model', '') == b.get('source_model', '')
|
|
row = {'symbol': s, 'mt5': a.get('source_model', ''), 'siamchart': b.get('source_model', ''),
|
|
'same': same, 'mt5_rank': lr.get(s), 'siamchart_rank': rr.get(s),
|
|
'rank_same': lr.get(s) == rr.get(s)}
|
|
for field in ('target_weight', 'allocation_score', 'reference_close'):
|
|
row['mt5_' + field] = a.get(field)
|
|
row['siamchart_' + field] = b.get(field)
|
|
row['same_' + field] = (field in a and field in b and
|
|
math.isclose(float(a[field]), float(b[field]), abs_tol=1e-6, rel_tol=1e-8))
|
|
signals.append(row)
|
|
set_left = left.set_close.reindex(days).to_numpy()
|
|
set_right = right.set_close.reindex(days).to_numpy()
|
|
set_mask = np.isfinite(set_left) & np.isfinite(set_right)
|
|
same_signals = all(s['same'] for s in signals)
|
|
latest_common = str(days[-1].date()) if len(days) else None
|
|
universe_left = set(left_features['universe'].iloc[-1].loc[lambda v: v].index)
|
|
universe_right = set(right_features['universe'].iloc[-1].loc[lambda v: v].index)
|
|
left_mask = left.has_bar.reindex(index=days, columns=symbols).to_numpy()
|
|
right_mask = right.has_bar.reindex(index=days, columns=symbols).to_numpy()
|
|
return plain({
|
|
'as_of': latest_common, 'same_signal_membership_and_model': same_signals,
|
|
'same_rankings': all(s['rank_same'] for s in signals),
|
|
'same_allocation_scores': all(s['same_allocation_score'] for s in signals),
|
|
'same_target_weights': all(s['same_target_weight'] for s in signals),
|
|
'shared_sessions': len(days), 'shared_symbols': len(symbols), 'ohlcv': fields,
|
|
'bars_only_mt5': int((left_mask & ~right_mask).sum()),
|
|
'bars_only_siamchart': int((right_mask & ~left_mask).sum()),
|
|
'sessions_only_mt5': [str(d.date()) for d in left.dates.difference(right.dates)],
|
|
'sessions_only_siamchart': [str(d.date()) for d in right.dates.difference(left.dates)],
|
|
'missing_from_mt5': missing_left, 'missing_from_siamchart': missing_right,
|
|
'set_compared': int(set_mask.sum()),
|
|
'set_different': int((set_mask & ~np.isclose(set_left, set_right, atol=1e-6, rtol=1e-8)).sum()),
|
|
'market_filter_mt5': {'close': float(left.set_close.iloc[-1]),
|
|
'sma200': float(left.set_close.rolling(200).mean().iloc[-1]),
|
|
'up': bool(left_features['market_up'].iloc[-1])},
|
|
'market_filter_siamchart': {'close': float(right.set_close.iloc[-1]),
|
|
'sma200': float(right.set_close.rolling(200).mean().iloc[-1]),
|
|
'up': bool(right_features['market_up'].iloc[-1])},
|
|
'signals': signals, 'mt5_count': len(ls), 'siamchart_count': len(rs),
|
|
'common_signal_count': len(set(ls) & set(rs)),
|
|
'latest_universe_mt5_only': sorted(universe_left - universe_right),
|
|
'latest_universe_siamchart_only': sorted(universe_right - universe_left),
|
|
'close_difference_samples': samples,
|
|
'notes': [
|
|
'ใช้กฎเดิม ช่วง warm-up เดียวกัน ข้อมูลปิดแล้ว และข้อมูลปันผล/แตกหุ้นชุดเดียวกัน',
|
|
'volume ใช้ real_volume เป็นจำนวนหุ้น ไม่ใช้ tick_volume แทน',
|
|
'MT5 แสดงหุ้นที่โบรกเกอร์มีในปัจจุบัน อาจไม่มีหุ้นที่เพิกถอนในประวัติเดิม',
|
|
'รายงานความต่างตามข้อมูลจริง ไม่มีการปรับราคา/volume ให้ดูตรงกัน',
|
|
],
|
|
})
|
|
|
|
|
|
def run_research(store, settings, progress):
|
|
progress('archive', 'เตรียมราคาจาก MT5 ที่เก็บในเครื่อง', 0, 1)
|
|
archive, through = export_archive(store, settings)
|
|
progress('signals', 'คำนวณ Donchian 55/20 และ EMA 50/200 จาก MT5', 0, 1)
|
|
current = panel_for([archive], settings, through)
|
|
features, signals, targets = calculate(current, settings)
|
|
output = {'updated_at': now_iso(), 'signal_as_of': str(current.dates[-1].date()),
|
|
'signals': signals, 'sessions': len(current.dates), 'symbols': len(current.symbols),
|
|
'history_issues': current.issues, 'warmup_start': settings.warmup_start,
|
|
'data_quality': store.get('data_quality', {}),
|
|
'model_start': settings.model_start,
|
|
'market_filter': {'set_close': float(current.set_close.iloc[-1]),
|
|
'sma200': float(current.set_close.rolling(200).mean().iloc[-1]),
|
|
'up': bool(features['market_up'].iloc[-1])},
|
|
'strategy_sha256': hashlib.sha256((REPO / 'primethai/model_lab.py').read_bytes()).hexdigest(),
|
|
'actions_note': 'ราคาจาก MT5; การปรับปันผล/แตกหุ้นใช้อ้างอิงที่มีอยู่ในโครงการเดิม',
|
|
'backtest_mt5_latest': backtest(current, targets, settings)}
|
|
archives = legacy_archives()
|
|
legacy_day = latest_archive_date(archives)
|
|
if legacy_day:
|
|
common_day = min(through, legacy_day)
|
|
progress('comparison', f'เทียบสองแหล่งข้อมูลถึงวันเดียวกัน {common_day}', 0, 1)
|
|
lp = panel_for([archive], settings, common_day)
|
|
rp = panel_for(archives, settings, common_day)
|
|
# Never label comparisons as equal-date if a whole market session is missing.
|
|
actual_common = min(lp.dates[-1], rp.dates[-1]).date().isoformat()
|
|
if str(lp.dates[-1].date()) != actual_common:
|
|
lp = panel_for([archive], settings, actual_common)
|
|
if str(rp.dates[-1].date()) != actual_common:
|
|
rp = panel_for(archives, settings, actual_common)
|
|
lf, ls, lt = calculate(lp, settings)
|
|
rf, rs, rt = calculate(rp, settings)
|
|
comparison = compare_panels(lp, rp, ls, rs, lf, rf)
|
|
comparison.update({'legacy_latest': legacy_day, 'mt5_latest': output['signal_as_of'],
|
|
'data_quality': output['data_quality'],
|
|
'backtest_mt5': backtest(lp, lt, settings),
|
|
'backtest_siamchart': backtest(rp, rt, settings)})
|
|
shared = lt.index.intersection(rt.index)
|
|
names = sorted(set(lt.columns) | set(rt.columns))
|
|
different_holdings = (lt.reindex(index=shared, columns=names, fill_value=0).gt(0)
|
|
!= rt.reindex(index=shared, columns=names, fill_value=0).gt(0))
|
|
comparison['historical_target_days_compared'] = len(shared[shared >= settings.model_start])
|
|
comparison['historical_target_days_different'] = int(different_holdings.loc[shared >= settings.model_start].any(axis=1).sum())
|
|
output['comparison'] = comparison
|
|
(settings.state_dir / 'comparison.json').write_text(json.dumps(plain(comparison), ensure_ascii=False, indent=2), encoding='utf-8')
|
|
pd.DataFrame(comparison['signals']).to_csv(settings.state_dir / 'signal_comparison.csv', index=False)
|
|
else:
|
|
output['comparison'] = {'error': 'ไม่พบไฟล์ SiamChart เดิมสำหรับเทียบ; สัญญาณหลักยังใช้ MT5'}
|
|
output = plain(output)
|
|
store.put('research', output)
|
|
(settings.state_dir / 'research.json').write_text(json.dumps(output, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
return output
|