"""Daily SET price snapshots from Yahoo Finance Chart API. This provider is for research plumbing only. Historical vendor responses are revision-prone and are deliberately marked ``point_in_time=false``. """ from __future__ import annotations import copy import hashlib import json import math import re from datetime import date, datetime, timedelta, timezone from pathlib import Path from typing import Any, Mapping from urllib.error import HTTPError, URLError from urllib.parse import urlencode from urllib.request import Request, build_opener from zoneinfo import ZoneInfo DEFAULT_SYMBOL_MAP = { "AOT": "AOT.BK", "MINT": "MINT.BK", "AWC": "AWC.BK", "CPN": "CPN.BK", "CPALL": "CPALL.BK", "CRC": "CRC.BK", "BEM": "BEM.BK", "PTT": "PTT.BK", "SET50": "^SET.BK", } PRICE_SCHEMA_VERSION = 1 PRICE_PARSER_VERSION = "yahoo-chart-v1" NORMALIZED_HASH_ALGORITHM = "sha256-json-canonical-v1" MAX_PAYLOAD_BYTES = 5_000_000 _PROVIDER_SYMBOL_RE = re.compile(r"^[A-Za-z0-9^._-]{1,64}$") class PriceSourceError(RuntimeError): """Raised when a price payload cannot be trusted or normalized.""" def _parse_date(value: str) -> date: try: return date.fromisoformat(value) except (TypeError, ValueError) as exc: raise PriceSourceError("price period must be YYYY-MM-DD") from exc def _combined_hash(raw_payloads: Mapping[str, bytes]) -> str: digest = hashlib.sha256() for symbol in sorted(raw_payloads): digest.update(symbol.encode("utf-8")) digest.update(b"\0") digest.update(raw_payloads[symbol]) digest.update(b"\0") return digest.hexdigest() def _normalized_snapshot_hash(snapshot: dict[str, Any]) -> str: payload = copy.deepcopy(snapshot) source = payload.get("source") if isinstance(source, dict): source.pop("normalized_snapshot_hash", None) encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") return hashlib.sha256(encoded).hexdigest() def _raw_filename(provider_symbol: str) -> str: if not _PROVIDER_SYMBOL_RE.fullmatch(provider_symbol): raise PriceSourceError("invalid provider symbol") safe = provider_symbol.replace("^", "index_").replace(".", "_") return f"{safe}.json" def _atomic_write(path: Path, content: bytes) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.tmp") temporary.write_bytes(content) temporary.replace(path) def normalize_yahoo_chart( payload: dict[str, Any], *, canonical_symbol: str, provider_symbol: str, retrieved_at: str, raw_payload_hash: str, ) -> dict[str, Any]: try: chart = payload["chart"] result = chart["result"][0] meta = result["meta"] timestamps = result["timestamp"] quote = result["indicators"]["quote"][0] adjusted = result["indicators"]["adjclose"][0]["adjclose"] except (KeyError, IndexError, TypeError) as exc: raise PriceSourceError("Yahoo chart result is missing required fields") from exc if not timestamps or not isinstance(adjusted, list): raise PriceSourceError("Yahoo chart result has no price rows") if meta.get("exchangeName") != "SET": raise PriceSourceError(f"unexpected exchange for {provider_symbol}") timezone_name = str(meta.get("exchangeTimezoneName") or "Asia/Bangkok") try: market_zone = ZoneInfo(timezone_name) except Exception as exc: raise PriceSourceError("Yahoo chart returned an unknown exchange timezone") from exc bars: list[dict[str, Any]] = [] seen_dates: set[str] = set() def at(values: Any, index: int) -> Any: return values[index] if isinstance(values, list) and index < len(values) else None opens = quote.get("open", []) highs = quote.get("high", []) lows = quote.get("low", []) closes = quote.get("close", []) volumes = quote.get("volume", []) for index, timestamp in enumerate(timestamps): raw_close = at(closes, index) raw_adjusted_close = at(adjusted, index) if timestamp is None or raw_close is None or raw_adjusted_close is None: continue try: timestamp_value = float(timestamp) close = float(raw_close) adjusted_close = float(raw_adjusted_close) except (KeyError, IndexError, TypeError, ValueError) as exc: raise PriceSourceError(f"invalid Yahoo price row for {provider_symbol}") from exc if not math.isfinite(timestamp_value) or not math.isfinite(close) or not math.isfinite(adjusted_close): continue if close <= 0 or adjusted_close <= 0: raise PriceSourceError(f"non-positive Yahoo close for {provider_symbol}") trading_date = datetime.fromtimestamp(timestamp_value, tz=timezone.utc).astimezone(market_zone).date().isoformat() if trading_date in seen_dates: raise PriceSourceError(f"duplicate Yahoo trading date for {provider_symbol}") seen_dates.add(trading_date) bar: dict[str, Any] = { "date": trading_date, "open": at(opens, index), "high": at(highs, index), "low": at(lows, index), "close": round(close, 8), "adjusted_close": round(adjusted_close, 8), "volume": at(volumes, index), } for key in ("open", "high", "low"): if bar[key] is not None: value = float(bar[key]) if not math.isfinite(value) or value <= 0: raise PriceSourceError(f"invalid Yahoo {key} for {provider_symbol}") bar[key] = round(value, 8) if bar["volume"] is not None: volume = float(bar["volume"]) if not math.isfinite(volume) or volume < 0: raise PriceSourceError(f"invalid Yahoo volume for {provider_symbol}") bar["volume"] = int(volume) bars.append(bar) if not bars: raise PriceSourceError(f"Yahoo chart has no usable rows for {provider_symbol}") bars.sort(key=lambda bar: bar["date"]) return { "canonical_symbol": canonical_symbol, "provider_symbol": provider_symbol, "exchange": meta["exchangeName"], "currency": meta.get("currency"), "timezone": timezone_name, "retrieved_at": retrieved_at, "raw_payload_hash": raw_payload_hash, "bars": bars, } def trading_dates(series: Mapping[str, Any]) -> list[str]: return [bar["date"] for bar in series.get("bars", [])] class YahooChartProvider: """Minimal public Yahoo chart API client with bounded response reads.""" def __init__(self, *, opener: Any | None = None, timeout: float = 30.0) -> None: self.opener = opener or build_opener() self.timeout = timeout def fetch_series(self, canonical_symbol: str, provider_symbol: str, start: str, end: str) -> tuple[dict[str, Any], bytes]: start_date = _parse_date(start) end_date = _parse_date(end) if end_date <= start_date: raise PriceSourceError("price end must be after price start") period1 = int(datetime.combine(start_date, datetime.min.time(), tzinfo=timezone.utc).timestamp()) period2 = int(datetime.combine(end_date + timedelta(days=1), datetime.min.time(), tzinfo=timezone.utc).timestamp()) query = urlencode({"period1": period1, "period2": period2, "interval": "1d", "events": "div,splits", "includeAdjustedClose": "true"}) url = f"https://query1.finance.yahoo.com/v8/finance/chart/{provider_symbol}?{query}" request = Request(url, headers={"User-Agent": "SET50-Alternative-Data-Platform/0.4"}) try: with self.opener.open(request, timeout=self.timeout) as response: if getattr(response, "status", 200) >= 400: raise PriceSourceError(f"Yahoo price source returned HTTP {response.status}") raw_bytes = response.read(MAX_PAYLOAD_BYTES + 1) except PriceSourceError: raise except (HTTPError, URLError, TimeoutError, OSError) as exc: raise PriceSourceError(f"Yahoo price source request failed: {exc.__class__.__name__}") from exc if len(raw_bytes) > MAX_PAYLOAD_BYTES: raise PriceSourceError("Yahoo price payload exceeds size limit") try: payload = json.loads(raw_bytes) except json.JSONDecodeError as exc: raise PriceSourceError("Yahoo price payload is not valid JSON") from exc retrieved_at = datetime.now(timezone.utc).isoformat() normalized = normalize_yahoo_chart( payload, canonical_symbol=canonical_symbol, provider_symbol=provider_symbol, retrieved_at=retrieved_at, raw_payload_hash=hashlib.sha256(raw_bytes).hexdigest(), ) return normalized, raw_bytes class PriceSnapshotStore: """Filesystem store for normalized price snapshots and raw provider payloads.""" def __init__(self, root: Path | str) -> None: self.root = Path(root).resolve() self.raw_dir = self.root / "raw" self.snapshot_dir = self.root / "snapshots" self.manifest_path = self.root / "manifest.json" def load_manifest(self) -> dict[str, Any]: if not self.manifest_path.is_file(): return {"schema_version": PRICE_SCHEMA_VERSION, "snapshots": {}} try: manifest = json.loads(self.manifest_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise PriceSourceError("price manifest is unreadable") from exc if manifest.get("schema_version") != PRICE_SCHEMA_VERSION or not isinstance(manifest.get("snapshots"), dict): raise PriceSourceError("unsupported price manifest schema") return manifest def _snapshot_path(self, snapshot_id: str) -> Path: if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}", snapshot_id): raise PriceSourceError("invalid price snapshot id") candidate = (self.snapshot_dir / f"{snapshot_id}.json").resolve() if candidate.parent != self.snapshot_dir.resolve(): raise PriceSourceError("price snapshot path escaped store root") return candidate def load_snapshot(self, snapshot_id: str) -> dict[str, Any]: snapshot_path = self._snapshot_path(snapshot_id) if not snapshot_path.is_file(): raise FileNotFoundError(snapshot_id) try: snapshot = json.loads(snapshot_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise PriceSourceError("price snapshot is unreadable") from exc source = snapshot.get("source") if not isinstance(source, dict) or source.get("snapshot_id") != snapshot_id: raise PriceSourceError("price snapshot identity mismatch") if not isinstance(source.get("point_in_time"), bool): raise PriceSourceError("price snapshot metadata point_in_time must be boolean") declared_snapshot_hash = source.get("normalized_snapshot_hash") if source.get("normalized_hash_algorithm") != NORMALIZED_HASH_ALGORITHM or not isinstance(declared_snapshot_hash, str) or _normalized_snapshot_hash(snapshot) != declared_snapshot_hash: raise PriceSourceError("price normalized snapshot hash mismatch") raw_files = source.get("raw_payload_files") if not isinstance(raw_files, dict) or not raw_files: raise PriceSourceError("price snapshot raw payload manifest is missing") raw_payloads: dict[str, bytes] = {} raw_root = (self.raw_dir / snapshot_id).resolve() if raw_root.parent != self.raw_dir.resolve(): raise PriceSourceError("price raw path escaped store root") for provider_symbol, filename in raw_files.items(): if _raw_filename(str(provider_symbol)) != filename: raise PriceSourceError("price raw filename mismatch") raw_path = (raw_root / filename).resolve() if raw_path.parent != raw_root or not raw_path.is_file(): raise PriceSourceError("price raw payload is missing") raw_payloads[str(provider_symbol)] = raw_path.read_bytes() actual_hash = _combined_hash(raw_payloads) if actual_hash != source.get("raw_payload_hash"): raise PriceSourceError("price raw payload hash mismatch") manifest_entry = self.load_manifest().get("snapshots", {}).get(snapshot_id) if not isinstance(manifest_entry, dict): raise PriceSourceError("price manifest entry is missing") if manifest_entry.get("source_id") != source.get("source_id") or manifest_entry.get("quality") != source.get("quality"): raise PriceSourceError("price manifest metadata mismatch") if manifest_entry.get("point_in_time") is not source.get("point_in_time"): raise PriceSourceError("price manifest metadata mismatch") if manifest_entry.get("raw_payload_hash") != actual_hash or manifest_entry.get("normalized_snapshot_hash") != declared_snapshot_hash or manifest_entry.get("normalized_hash_algorithm") != NORMALIZED_HASH_ALGORITHM: raise PriceSourceError("price manifest hash mismatch") return snapshot def list_snapshots(self) -> list[dict[str, Any]]: entries = list(self.load_manifest().get("snapshots", {}).values()) return sorted(entries, key=lambda item: (str(item.get("retrieved_at", "")), str(item.get("snapshot_id", "")))) def latest_snapshot_entry(self) -> dict[str, Any] | None: entries = self.list_snapshots() return entries[-1] if entries else None def persist(self, snapshot: dict[str, Any], raw_payloads: Mapping[str, bytes]) -> dict[str, Any]: source = snapshot.get("source") if not isinstance(source, dict) or not source.get("snapshot_id"): raise PriceSourceError("price snapshot source metadata is required") if not raw_payloads or any(not isinstance(value, bytes) or not value for value in raw_payloads.values()): raise PriceSourceError("price raw payloads must be non-empty bytes") raw_hash = _combined_hash(raw_payloads) if source.get("raw_payload_hash") and source["raw_payload_hash"] != raw_hash: raise PriceSourceError("raw payload hash does not match price snapshot") if "point_in_time" in source and not isinstance(source["point_in_time"], bool): raise PriceSourceError("price snapshot metadata point_in_time must be boolean") stored = copy.deepcopy(snapshot) stored_source = stored["source"] snapshot_id = str(stored_source["snapshot_id"]) if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}", snapshot_id): raise PriceSourceError("invalid price snapshot id") stored_source["raw_payload_hash"] = raw_hash stored_source["raw_payload_files"] = {symbol: _raw_filename(symbol) for symbol in raw_payloads} stored_source["snapshot_file"] = f"{snapshot_id}.json" stored_source["normalized_hash_algorithm"] = NORMALIZED_HASH_ALGORITHM stored_source["normalized_snapshot_hash"] = _normalized_snapshot_hash(stored) for provider_symbol, payload in raw_payloads.items(): _atomic_write(self.raw_dir / snapshot_id / _raw_filename(provider_symbol), payload) _atomic_write(self.snapshot_dir / stored_source["snapshot_file"], (json.dumps(stored, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8")) manifest = self.load_manifest() manifest["snapshots"][snapshot_id] = { "snapshot_id": snapshot_id, "source_id": stored_source.get("source_id"), "retrieved_at": stored_source.get("retrieved_at"), "period_start": stored_source.get("period_start"), "period_end": stored_source.get("period_end"), "raw_payload_hash": raw_hash, "normalized_snapshot_hash": stored_source["normalized_snapshot_hash"], "normalized_hash_algorithm": NORMALIZED_HASH_ALGORITHM, "parser_version": stored_source.get("parser_version"), "quality": stored_source.get("quality"), "point_in_time": stored_source.get("point_in_time"), "adjusted_prices": stored_source.get("adjusted_prices"), "return_price_field": stored_source.get("return_price_field"), "symbols": sorted(stored.get("series", {}).keys()), "snapshot_file": stored_source["snapshot_file"], } _atomic_write(self.manifest_path, (json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8")) return stored def collect_price_snapshot( root: Path | str, *, start: str, end: str, symbol_map: Mapping[str, str] = DEFAULT_SYMBOL_MAP, provider: YahooChartProvider | Any | None = None, retrieved_at: str | None = None, ) -> dict[str, Any]: if not symbol_map: raise PriceSourceError("symbol_map must not be empty") provider = provider or YahooChartProvider() normalized: dict[str, Any] = {} raw_payloads: dict[str, bytes] = {} for canonical_symbol, provider_symbol in symbol_map.items(): series, raw_bytes = provider.fetch_series(canonical_symbol, provider_symbol, start, end) normalized[canonical_symbol] = series raw_payloads[provider_symbol] = raw_bytes raw_hash = _combined_hash(raw_payloads) retrieved = retrieved_at or datetime.now(timezone.utc).isoformat() snapshot_id = f"prices-yahoo-chart-{start}-{end}-{raw_hash[:12]}" source = { "source_id": "yahoo.chart", "source_url": "https://query1.finance.yahoo.com/v8/finance/chart/{provider_symbol}", "retrieved_at": retrieved, "period_start": start, "period_end": end, "raw_payload_hash": raw_hash, "parser_version": PRICE_PARSER_VERSION, "quality": "revised_vendor_history", "point_in_time": False, "adjusted_prices": True, "return_price_field": "close", "snapshot_id": snapshot_id, "bar_counts": {symbol: len(series["bars"]) for symbol, series in normalized.items()}, } snapshot = {"schema_version": PRICE_SCHEMA_VERSION, "source": source, "series": normalized, "benchmark_symbol": "SET50" if "SET50" in normalized else None} return PriceSnapshotStore(root).persist(snapshot, raw_payloads)