360 lines
16 KiB
Python
360 lines
16 KiB
Python
"""Daily SET price snapshots from Yahoo Finance Chart API.
|
|
|
|
This provider is for research plumbing only. Historical vendor responses are
|
|
revision-prone and are deliberately marked ``point_in_time=false``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import re
|
|
from datetime import date, datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Mapping
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urlencode
|
|
from urllib.request import Request, build_opener
|
|
from zoneinfo import ZoneInfo
|
|
|
|
DEFAULT_SYMBOL_MAP = {
|
|
"AOT": "AOT.BK",
|
|
"MINT": "MINT.BK",
|
|
"AWC": "AWC.BK",
|
|
"CPN": "CPN.BK",
|
|
"CPALL": "CPALL.BK",
|
|
"CRC": "CRC.BK",
|
|
"BEM": "BEM.BK",
|
|
"PTT": "PTT.BK",
|
|
"SET50": "^SET.BK",
|
|
}
|
|
PRICE_SCHEMA_VERSION = 1
|
|
PRICE_PARSER_VERSION = "yahoo-chart-v1"
|
|
MAX_PAYLOAD_BYTES = 5_000_000
|
|
_PROVIDER_SYMBOL_RE = re.compile(r"^[A-Za-z0-9^._-]{1,64}$")
|
|
|
|
|
|
class PriceSourceError(RuntimeError):
|
|
"""Raised when a price payload cannot be trusted or normalized."""
|
|
|
|
|
|
def _parse_date(value: str) -> date:
|
|
try:
|
|
return date.fromisoformat(value)
|
|
except (TypeError, ValueError) as exc:
|
|
raise PriceSourceError("price period must be YYYY-MM-DD") from exc
|
|
|
|
|
|
def _combined_hash(raw_payloads: Mapping[str, bytes]) -> str:
|
|
digest = hashlib.sha256()
|
|
for symbol in sorted(raw_payloads):
|
|
digest.update(symbol.encode("utf-8"))
|
|
digest.update(b"\0")
|
|
digest.update(raw_payloads[symbol])
|
|
digest.update(b"\0")
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _raw_filename(provider_symbol: str) -> str:
|
|
if not _PROVIDER_SYMBOL_RE.fullmatch(provider_symbol):
|
|
raise PriceSourceError("invalid provider symbol")
|
|
safe = provider_symbol.replace("^", "index_").replace(".", "_")
|
|
return f"{safe}.json"
|
|
|
|
|
|
def _atomic_write(path: Path, content: bytes) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_name(f".{path.name}.tmp")
|
|
temporary.write_bytes(content)
|
|
temporary.replace(path)
|
|
|
|
|
|
def normalize_yahoo_chart(
|
|
payload: dict[str, Any],
|
|
*,
|
|
canonical_symbol: str,
|
|
provider_symbol: str,
|
|
retrieved_at: str,
|
|
raw_payload_hash: str,
|
|
) -> dict[str, Any]:
|
|
try:
|
|
chart = payload["chart"]
|
|
result = chart["result"][0]
|
|
meta = result["meta"]
|
|
timestamps = result["timestamp"]
|
|
quote = result["indicators"]["quote"][0]
|
|
adjusted = result["indicators"]["adjclose"][0]["adjclose"]
|
|
except (KeyError, IndexError, TypeError) as exc:
|
|
raise PriceSourceError("Yahoo chart result is missing required fields") from exc
|
|
if not timestamps or not isinstance(adjusted, list):
|
|
raise PriceSourceError("Yahoo chart result has no price rows")
|
|
if meta.get("exchangeName") != "SET":
|
|
raise PriceSourceError(f"unexpected exchange for {provider_symbol}")
|
|
timezone_name = str(meta.get("exchangeTimezoneName") or "Asia/Bangkok")
|
|
try:
|
|
market_zone = ZoneInfo(timezone_name)
|
|
except Exception as exc:
|
|
raise PriceSourceError("Yahoo chart returned an unknown exchange timezone") from exc
|
|
|
|
bars: list[dict[str, Any]] = []
|
|
seen_dates: set[str] = set()
|
|
def at(values: Any, index: int) -> Any:
|
|
return values[index] if isinstance(values, list) and index < len(values) else None
|
|
|
|
opens = quote.get("open", [])
|
|
highs = quote.get("high", [])
|
|
lows = quote.get("low", [])
|
|
closes = quote.get("close", [])
|
|
volumes = quote.get("volume", [])
|
|
for index, timestamp in enumerate(timestamps):
|
|
raw_close = at(closes, index)
|
|
raw_adjusted_close = at(adjusted, index)
|
|
if timestamp is None or raw_close is None or raw_adjusted_close is None:
|
|
continue
|
|
try:
|
|
timestamp_value = float(timestamp)
|
|
close = float(raw_close)
|
|
adjusted_close = float(raw_adjusted_close)
|
|
except (KeyError, IndexError, TypeError, ValueError) as exc:
|
|
raise PriceSourceError(f"invalid Yahoo price row for {provider_symbol}") from exc
|
|
if not math.isfinite(timestamp_value) or not math.isfinite(close) or not math.isfinite(adjusted_close):
|
|
continue
|
|
if close <= 0 or adjusted_close <= 0:
|
|
raise PriceSourceError(f"non-positive Yahoo close for {provider_symbol}")
|
|
trading_date = datetime.fromtimestamp(timestamp_value, tz=timezone.utc).astimezone(market_zone).date().isoformat()
|
|
if trading_date in seen_dates:
|
|
raise PriceSourceError(f"duplicate Yahoo trading date for {provider_symbol}")
|
|
seen_dates.add(trading_date)
|
|
bar: dict[str, Any] = {
|
|
"date": trading_date,
|
|
"open": at(opens, index),
|
|
"high": at(highs, index),
|
|
"low": at(lows, index),
|
|
"close": round(close, 8),
|
|
"adjusted_close": round(adjusted_close, 8),
|
|
"volume": at(volumes, index),
|
|
}
|
|
for key in ("open", "high", "low"):
|
|
if bar[key] is not None:
|
|
value = float(bar[key])
|
|
if not math.isfinite(value) or value <= 0:
|
|
raise PriceSourceError(f"invalid Yahoo {key} for {provider_symbol}")
|
|
bar[key] = round(value, 8)
|
|
if bar["volume"] is not None:
|
|
volume = float(bar["volume"])
|
|
if not math.isfinite(volume) or volume < 0:
|
|
raise PriceSourceError(f"invalid Yahoo volume for {provider_symbol}")
|
|
bar["volume"] = int(volume)
|
|
bars.append(bar)
|
|
if not bars:
|
|
raise PriceSourceError(f"Yahoo chart has no usable rows for {provider_symbol}")
|
|
bars.sort(key=lambda bar: bar["date"])
|
|
return {
|
|
"canonical_symbol": canonical_symbol,
|
|
"provider_symbol": provider_symbol,
|
|
"exchange": meta["exchangeName"],
|
|
"currency": meta.get("currency"),
|
|
"timezone": timezone_name,
|
|
"retrieved_at": retrieved_at,
|
|
"raw_payload_hash": raw_payload_hash,
|
|
"bars": bars,
|
|
}
|
|
|
|
|
|
def trading_dates(series: Mapping[str, Any]) -> list[str]:
|
|
return [bar["date"] for bar in series.get("bars", [])]
|
|
|
|
|
|
class YahooChartProvider:
|
|
"""Minimal public Yahoo chart API client with bounded response reads."""
|
|
|
|
def __init__(self, *, opener: Any | None = None, timeout: float = 30.0) -> None:
|
|
self.opener = opener or build_opener()
|
|
self.timeout = timeout
|
|
|
|
def fetch_series(self, canonical_symbol: str, provider_symbol: str, start: str, end: str) -> tuple[dict[str, Any], bytes]:
|
|
start_date = _parse_date(start)
|
|
end_date = _parse_date(end)
|
|
if end_date <= start_date:
|
|
raise PriceSourceError("price end must be after price start")
|
|
period1 = int(datetime.combine(start_date, datetime.min.time(), tzinfo=timezone.utc).timestamp())
|
|
period2 = int(datetime.combine(end_date + timedelta(days=1), datetime.min.time(), tzinfo=timezone.utc).timestamp())
|
|
query = urlencode({"period1": period1, "period2": period2, "interval": "1d", "events": "div,splits", "includeAdjustedClose": "true"})
|
|
url = f"https://query1.finance.yahoo.com/v8/finance/chart/{provider_symbol}?{query}"
|
|
request = Request(url, headers={"User-Agent": "SET50-Alternative-Data-Platform/0.4"})
|
|
try:
|
|
with self.opener.open(request, timeout=self.timeout) as response:
|
|
if getattr(response, "status", 200) >= 400:
|
|
raise PriceSourceError(f"Yahoo price source returned HTTP {response.status}")
|
|
raw_bytes = response.read(MAX_PAYLOAD_BYTES + 1)
|
|
except PriceSourceError:
|
|
raise
|
|
except (HTTPError, URLError, TimeoutError, OSError) as exc:
|
|
raise PriceSourceError(f"Yahoo price source request failed: {exc.__class__.__name__}") from exc
|
|
if len(raw_bytes) > MAX_PAYLOAD_BYTES:
|
|
raise PriceSourceError("Yahoo price payload exceeds size limit")
|
|
try:
|
|
payload = json.loads(raw_bytes)
|
|
except json.JSONDecodeError as exc:
|
|
raise PriceSourceError("Yahoo price payload is not valid JSON") from exc
|
|
retrieved_at = datetime.now(timezone.utc).isoformat()
|
|
normalized = normalize_yahoo_chart(
|
|
payload,
|
|
canonical_symbol=canonical_symbol,
|
|
provider_symbol=provider_symbol,
|
|
retrieved_at=retrieved_at,
|
|
raw_payload_hash=hashlib.sha256(raw_bytes).hexdigest(),
|
|
)
|
|
return normalized, raw_bytes
|
|
|
|
|
|
class PriceSnapshotStore:
|
|
"""Filesystem store for normalized price snapshots and raw provider payloads."""
|
|
|
|
def __init__(self, root: Path | str) -> None:
|
|
self.root = Path(root).resolve()
|
|
self.raw_dir = self.root / "raw"
|
|
self.snapshot_dir = self.root / "snapshots"
|
|
self.manifest_path = self.root / "manifest.json"
|
|
|
|
def load_manifest(self) -> dict[str, Any]:
|
|
if not self.manifest_path.is_file():
|
|
return {"schema_version": PRICE_SCHEMA_VERSION, "snapshots": {}}
|
|
try:
|
|
manifest = json.loads(self.manifest_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise PriceSourceError("price manifest is unreadable") from exc
|
|
if manifest.get("schema_version") != PRICE_SCHEMA_VERSION or not isinstance(manifest.get("snapshots"), dict):
|
|
raise PriceSourceError("unsupported price manifest schema")
|
|
return manifest
|
|
|
|
def _snapshot_path(self, snapshot_id: str) -> Path:
|
|
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}", snapshot_id):
|
|
raise PriceSourceError("invalid price snapshot id")
|
|
candidate = (self.snapshot_dir / f"{snapshot_id}.json").resolve()
|
|
if candidate.parent != self.snapshot_dir.resolve():
|
|
raise PriceSourceError("price snapshot path escaped store root")
|
|
return candidate
|
|
|
|
def load_snapshot(self, snapshot_id: str) -> dict[str, Any]:
|
|
snapshot_path = self._snapshot_path(snapshot_id)
|
|
if not snapshot_path.is_file():
|
|
raise FileNotFoundError(snapshot_id)
|
|
try:
|
|
snapshot = json.loads(snapshot_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise PriceSourceError("price snapshot is unreadable") from exc
|
|
source = snapshot.get("source")
|
|
if not isinstance(source, dict) or source.get("snapshot_id") != snapshot_id:
|
|
raise PriceSourceError("price snapshot identity mismatch")
|
|
raw_files = source.get("raw_payload_files")
|
|
if not isinstance(raw_files, dict) or not raw_files:
|
|
raise PriceSourceError("price snapshot raw payload manifest is missing")
|
|
raw_payloads: dict[str, bytes] = {}
|
|
raw_root = (self.raw_dir / snapshot_id).resolve()
|
|
if raw_root.parent != self.raw_dir.resolve():
|
|
raise PriceSourceError("price raw path escaped store root")
|
|
for provider_symbol, filename in raw_files.items():
|
|
if _raw_filename(str(provider_symbol)) != filename:
|
|
raise PriceSourceError("price raw filename mismatch")
|
|
raw_path = (raw_root / filename).resolve()
|
|
if raw_path.parent != raw_root or not raw_path.is_file():
|
|
raise PriceSourceError("price raw payload is missing")
|
|
raw_payloads[str(provider_symbol)] = raw_path.read_bytes()
|
|
actual_hash = _combined_hash(raw_payloads)
|
|
if actual_hash != source.get("raw_payload_hash"):
|
|
raise PriceSourceError("price raw payload hash mismatch")
|
|
manifest_entry = self.load_manifest().get("snapshots", {}).get(snapshot_id)
|
|
if isinstance(manifest_entry, dict) and manifest_entry.get("raw_payload_hash") != actual_hash:
|
|
raise PriceSourceError("price manifest hash mismatch")
|
|
return snapshot
|
|
|
|
def list_snapshots(self) -> list[dict[str, Any]]:
|
|
entries = list(self.load_manifest().get("snapshots", {}).values())
|
|
return sorted(entries, key=lambda item: (str(item.get("retrieved_at", "")), str(item.get("snapshot_id", ""))))
|
|
|
|
def latest_snapshot_entry(self) -> dict[str, Any] | None:
|
|
entries = self.list_snapshots()
|
|
return entries[-1] if entries else None
|
|
|
|
def persist(self, snapshot: dict[str, Any], raw_payloads: Mapping[str, bytes]) -> dict[str, Any]:
|
|
source = snapshot.get("source")
|
|
if not isinstance(source, dict) or not source.get("snapshot_id"):
|
|
raise PriceSourceError("price snapshot source metadata is required")
|
|
if not raw_payloads or any(not isinstance(value, bytes) or not value for value in raw_payloads.values()):
|
|
raise PriceSourceError("price raw payloads must be non-empty bytes")
|
|
raw_hash = _combined_hash(raw_payloads)
|
|
if source.get("raw_payload_hash") and source["raw_payload_hash"] != raw_hash:
|
|
raise PriceSourceError("raw payload hash does not match price snapshot")
|
|
stored = copy.deepcopy(snapshot)
|
|
stored_source = stored["source"]
|
|
snapshot_id = str(stored_source["snapshot_id"])
|
|
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}", snapshot_id):
|
|
raise PriceSourceError("invalid price snapshot id")
|
|
stored_source["raw_payload_hash"] = raw_hash
|
|
stored_source["raw_payload_files"] = {symbol: _raw_filename(symbol) for symbol in raw_payloads}
|
|
stored_source["snapshot_file"] = f"{snapshot_id}.json"
|
|
for provider_symbol, payload in raw_payloads.items():
|
|
_atomic_write(self.raw_dir / snapshot_id / _raw_filename(provider_symbol), payload)
|
|
_atomic_write(self.snapshot_dir / stored_source["snapshot_file"], (json.dumps(stored, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8"))
|
|
manifest = self.load_manifest()
|
|
manifest["snapshots"][snapshot_id] = {
|
|
"snapshot_id": snapshot_id,
|
|
"source_id": stored_source.get("source_id"),
|
|
"retrieved_at": stored_source.get("retrieved_at"),
|
|
"period_start": stored_source.get("period_start"),
|
|
"period_end": stored_source.get("period_end"),
|
|
"raw_payload_hash": raw_hash,
|
|
"parser_version": stored_source.get("parser_version"),
|
|
"quality": stored_source.get("quality"),
|
|
"point_in_time": stored_source.get("point_in_time"),
|
|
"adjusted_prices": stored_source.get("adjusted_prices"),
|
|
"return_price_field": stored_source.get("return_price_field"),
|
|
"symbols": sorted(stored.get("series", {}).keys()),
|
|
"snapshot_file": stored_source["snapshot_file"],
|
|
}
|
|
_atomic_write(self.manifest_path, (json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8"))
|
|
return stored
|
|
|
|
|
|
def collect_price_snapshot(
|
|
root: Path | str,
|
|
*,
|
|
start: str,
|
|
end: str,
|
|
symbol_map: Mapping[str, str] = DEFAULT_SYMBOL_MAP,
|
|
provider: YahooChartProvider | Any | None = None,
|
|
retrieved_at: str | None = None,
|
|
) -> dict[str, Any]:
|
|
if not symbol_map:
|
|
raise PriceSourceError("symbol_map must not be empty")
|
|
provider = provider or YahooChartProvider()
|
|
normalized: dict[str, Any] = {}
|
|
raw_payloads: dict[str, bytes] = {}
|
|
for canonical_symbol, provider_symbol in symbol_map.items():
|
|
series, raw_bytes = provider.fetch_series(canonical_symbol, provider_symbol, start, end)
|
|
normalized[canonical_symbol] = series
|
|
raw_payloads[provider_symbol] = raw_bytes
|
|
raw_hash = _combined_hash(raw_payloads)
|
|
retrieved = retrieved_at or datetime.now(timezone.utc).isoformat()
|
|
snapshot_id = f"prices-yahoo-chart-{start}-{end}-{raw_hash[:12]}"
|
|
source = {
|
|
"source_id": "yahoo.chart",
|
|
"source_url": "https://query1.finance.yahoo.com/v8/finance/chart/{provider_symbol}",
|
|
"retrieved_at": retrieved,
|
|
"period_start": start,
|
|
"period_end": end,
|
|
"raw_payload_hash": raw_hash,
|
|
"parser_version": PRICE_PARSER_VERSION,
|
|
"quality": "revised_vendor_history",
|
|
"point_in_time": False,
|
|
"adjusted_prices": True,
|
|
"return_price_field": "close",
|
|
"snapshot_id": snapshot_id,
|
|
"bar_counts": {symbol: len(series["bars"]) for symbol, series in normalized.items()},
|
|
}
|
|
snapshot = {"schema_version": PRICE_SCHEMA_VERSION, "source": source, "series": normalized, "benchmark_symbol": "SET50" if "SET50" in normalized else None}
|
|
return PriceSnapshotStore(root).persist(snapshot, raw_payloads)
|