f6dd6304cc
- auto_credit.py: scrape Trading Economics Thailand total-vehicle-sales HTML -> total sales + new car sales YoY + vehicle production/passenger/exports - refining_energy.py: scrape EIA prices.php 3:2:1 crack spread (Gulf LLS) + WTI/Brent/gasoline - Both server-rendered HTML scrapers (har-derived-api-client pattern); EIA used instead of RBN (RBN is Cloudflare-challenged, 403 via urllib; EIA is open US gov data, HTTP 200 direct) - collect_auto_credit.py / collect_refining.py CLI -> JSON snapshot - 8 tests; full backend suite 148 OK; live verified (auto 59198/20% YoY; crack 71.10 $/bbl); static scan clean
39 lines
1.1 KiB
Python
39 lines
1.1 KiB
Python
"""Collect the Refining/Energy factor (EIA crack spread) to a JSON snapshot."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from app import refining_energy
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--out", type=Path,
|
|
help="write JSON snapshot to this file (default: stdout)")
|
|
parser.add_argument("--timeout", type=float, default=30.0)
|
|
args = parser.parse_args()
|
|
|
|
snap = refining_energy.fetch_refining(timeout=args.timeout)
|
|
payload = {
|
|
"source": "eia",
|
|
"retrieved_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"factor": "refining_energy",
|
|
"data": snap.to_dict(),
|
|
}
|
|
text = json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2)
|
|
if args.out:
|
|
args.out.parent.mkdir(parents=True, exist_ok=True)
|
|
args.out.write_text(text, encoding="utf-8")
|
|
print(f"wrote refining_energy snapshot -> {args.out}")
|
|
else:
|
|
print(text)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|