201 lines
9.8 KiB
Python
201 lines
9.8 KiB
Python
"""Immutable tourism vintage manifest and point-in-time helpers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Iterable
|
|
|
|
MANIFEST_SCHEMA_VERSION = 1
|
|
NORMALIZED_HASH_ALGORITHM = "sha256-json-canonical-v1"
|
|
_VINTAGE_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}")
|
|
_FILENAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,191}\.(?:html|json)")
|
|
|
|
|
|
class VintageStoreError(ValueError):
|
|
"""Raised when a vintage cannot be safely stored or loaded."""
|
|
|
|
|
|
def _parse_timestamp(value: str) -> datetime:
|
|
try:
|
|
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
except (TypeError, ValueError) as exc:
|
|
raise VintageStoreError("timestamp must be ISO-8601") from exc
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
return parsed
|
|
|
|
|
|
def _normalized_snapshot_hash(snapshot: dict[str, Any]) -> str:
|
|
payload = copy.deepcopy(snapshot)
|
|
source = payload.get("source")
|
|
if isinstance(source, dict):
|
|
source.pop("normalized_snapshot_hash", None)
|
|
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
return hashlib.sha256(encoded).hexdigest()
|
|
|
|
|
|
def _safe_vintage_id(vintage_id: str) -> str:
|
|
if not _VINTAGE_ID_RE.fullmatch(vintage_id):
|
|
raise VintageStoreError("invalid vintage_id")
|
|
return vintage_id
|
|
|
|
|
|
def _safe_filename(filename: str, suffix: str) -> str:
|
|
if not _FILENAME_RE.fullmatch(filename) or not filename.endswith(suffix):
|
|
raise VintageStoreError("invalid vintage filename")
|
|
if Path(filename).name != filename:
|
|
raise VintageStoreError("vintage filename must not contain a path")
|
|
return filename
|
|
|
|
|
|
def filter_vintages(entries: Iterable[dict[str, Any]], as_of: str | None = None) -> list[dict[str, Any]]:
|
|
"""Return vintages whose publication timestamp was known by ``as_of``."""
|
|
|
|
cutoff = _parse_timestamp(as_of) if as_of else None
|
|
selected = []
|
|
for entry in entries:
|
|
published_at = _parse_timestamp(str(entry["published_at"]))
|
|
if cutoff is None or published_at <= cutoff:
|
|
selected.append(copy.deepcopy(entry))
|
|
return sorted(selected, key=lambda item: (_parse_timestamp(str(item["published_at"])), item["vintage_id"]))
|
|
|
|
|
|
def _atomic_write_bytes(path: Path, content: bytes) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_name(f".{path.name}.tmp")
|
|
temporary.write_bytes(content)
|
|
temporary.replace(path)
|
|
|
|
|
|
def _atomic_write_json(path: Path, payload: dict[str, Any]) -> None:
|
|
_atomic_write_bytes(path, (json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8"))
|
|
|
|
|
|
class VintageStore:
|
|
"""Filesystem-backed immutable raw/snapshot store with a compact manifest."""
|
|
|
|
def __init__(self, root: Path | str) -> None:
|
|
self.root = Path(root).resolve()
|
|
self.raw_dir = self.root / "raw" / "tourism"
|
|
self.snapshot_dir = self.root / "snapshots"
|
|
self.manifest_path = self.root / "manifest.json"
|
|
|
|
def load_manifest(self) -> dict[str, Any]:
|
|
if not self.manifest_path.is_file():
|
|
return {"schema_version": MANIFEST_SCHEMA_VERSION, "vintages": {}}
|
|
try:
|
|
payload = json.loads(self.manifest_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise VintageStoreError("vintage manifest is unreadable") from exc
|
|
if not isinstance(payload, dict) or payload.get("schema_version") != MANIFEST_SCHEMA_VERSION:
|
|
raise VintageStoreError("unsupported vintage manifest schema")
|
|
if not isinstance(payload.get("vintages"), dict):
|
|
raise VintageStoreError("vintage manifest has invalid vintages map")
|
|
return payload
|
|
|
|
def persist(self, raw_bytes: bytes, snapshot: dict[str, Any]) -> dict[str, Any]:
|
|
if not isinstance(raw_bytes, bytes) or not raw_bytes:
|
|
raise VintageStoreError("raw vintage payload must be non-empty bytes")
|
|
stored_snapshot = copy.deepcopy(snapshot)
|
|
source = stored_snapshot.get("source")
|
|
if not isinstance(source, dict):
|
|
raise VintageStoreError("snapshot source metadata is required")
|
|
vintage_id = _safe_vintage_id(str(source.get("vintage_id", "")))
|
|
published_at = str(source.get("published_at", ""))
|
|
retrieved_at = str(source.get("retrieved_at", ""))
|
|
_parse_timestamp(published_at)
|
|
_parse_timestamp(retrieved_at)
|
|
raw_hash = hashlib.sha256(raw_bytes).hexdigest()
|
|
declared_hash = source.get("raw_payload_hash")
|
|
if declared_hash and declared_hash != raw_hash:
|
|
raise VintageStoreError("raw payload hash does not match snapshot metadata")
|
|
source["raw_payload_hash"] = raw_hash
|
|
source["raw_snapshot_file"] = _safe_filename(
|
|
str(source.get("raw_snapshot_file") or f"{vintage_id}.html"), ".html"
|
|
)
|
|
source["snapshot_file"] = _safe_filename(
|
|
str(source.get("snapshot_file") or f"{vintage_id}.json"), ".json"
|
|
)
|
|
|
|
manifest = self.load_manifest()
|
|
existing = manifest["vintages"].get(vintage_id)
|
|
same_release = any(
|
|
item.get("source_id") == source.get("source_id")
|
|
and _parse_timestamp(str(item.get("published_at"))) == _parse_timestamp(published_at)
|
|
and item.get("vintage_id") != vintage_id
|
|
for item in manifest["vintages"].values()
|
|
)
|
|
revision_status = existing.get("revision_status") if existing else ("revised" if same_release else "initial")
|
|
source["revision_status"] = revision_status
|
|
source["normalized_hash_algorithm"] = NORMALIZED_HASH_ALGORITHM
|
|
stored_snapshot["source"] = source
|
|
source["normalized_snapshot_hash"] = _normalized_snapshot_hash(stored_snapshot)
|
|
|
|
_atomic_write_bytes(self.raw_dir / source["raw_snapshot_file"], raw_bytes)
|
|
_atomic_write_json(self.snapshot_dir / source["snapshot_file"], stored_snapshot)
|
|
|
|
first_seen_at = existing.get("first_seen_at", retrieved_at) if existing else retrieved_at
|
|
last_seen_at = max(_parse_timestamp(first_seen_at), _parse_timestamp(retrieved_at)).isoformat()
|
|
manifest["vintages"][vintage_id] = {
|
|
"vintage_id": vintage_id,
|
|
"source_id": source.get("source_id"),
|
|
"source_url": source.get("source_url"),
|
|
"as_of": stored_snapshot.get("as_of"),
|
|
"published_at": published_at,
|
|
"release_status": source.get("release_status"),
|
|
"revision_status": revision_status,
|
|
"raw_payload_hash": raw_hash,
|
|
"normalized_snapshot_hash": source["normalized_snapshot_hash"],
|
|
"normalized_hash_algorithm": NORMALIZED_HASH_ALGORITHM,
|
|
"parser_version": source.get("parser_version"),
|
|
"raw_snapshot_file": source["raw_snapshot_file"],
|
|
"snapshot_file": source["snapshot_file"],
|
|
"first_seen_at": first_seen_at,
|
|
"last_seen_at": last_seen_at,
|
|
"seen_count": int(existing.get("seen_count", 0)) + 1 if existing else 1,
|
|
}
|
|
_atomic_write_json(self.manifest_path, manifest)
|
|
return stored_snapshot
|
|
|
|
def list_vintages(self, as_of: str | None = None) -> list[dict[str, Any]]:
|
|
return filter_vintages(self.load_manifest()["vintages"].values(), as_of)
|
|
|
|
def load_snapshot(self, vintage_id: str) -> dict[str, Any]:
|
|
vintage_id = _safe_vintage_id(vintage_id)
|
|
snapshot_path = (self.snapshot_dir / f"{vintage_id}.json").resolve()
|
|
if snapshot_path.parent != self.snapshot_dir.resolve():
|
|
raise VintageStoreError("snapshot path escaped store root")
|
|
if not snapshot_path.is_file():
|
|
raise FileNotFoundError(vintage_id)
|
|
try:
|
|
snapshot = json.loads(snapshot_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise VintageStoreError("vintage snapshot is unreadable") from exc
|
|
source = snapshot.get("source")
|
|
if not isinstance(source, dict) or source.get("vintage_id") != vintage_id:
|
|
raise VintageStoreError("vintage snapshot identity mismatch")
|
|
declared_snapshot_hash = source.get("normalized_snapshot_hash")
|
|
if source.get("normalized_hash_algorithm") != NORMALIZED_HASH_ALGORITHM or not isinstance(declared_snapshot_hash, str) or _normalized_snapshot_hash(snapshot) != declared_snapshot_hash:
|
|
raise VintageStoreError("vintage normalized snapshot hash mismatch")
|
|
raw_filename = _safe_filename(str(source.get("raw_snapshot_file", "")), ".html")
|
|
raw_path = (self.raw_dir / raw_filename).resolve()
|
|
if raw_path.parent != self.raw_dir.resolve() or not raw_path.is_file():
|
|
raise VintageStoreError("vintage raw payload is missing")
|
|
if hashlib.sha256(raw_path.read_bytes()).hexdigest() != source.get("raw_payload_hash"):
|
|
raise VintageStoreError("vintage raw payload hash mismatch")
|
|
manifest_entry = self.load_manifest().get("vintages", {}).get(vintage_id)
|
|
if not isinstance(manifest_entry, dict):
|
|
raise VintageStoreError("vintage manifest entry is missing")
|
|
if manifest_entry.get("source_id") != source.get("source_id"):
|
|
raise VintageStoreError("vintage manifest metadata mismatch")
|
|
if _parse_timestamp(str(manifest_entry.get("published_at"))) != _parse_timestamp(str(source.get("published_at"))):
|
|
raise VintageStoreError("vintage manifest metadata mismatch")
|
|
if manifest_entry.get("raw_payload_hash") != source.get("raw_payload_hash") or manifest_entry.get("normalized_snapshot_hash") != declared_snapshot_hash or manifest_entry.get("normalized_hash_algorithm") != NORMALIZED_HASH_ALGORITHM:
|
|
raise VintageStoreError("vintage manifest hash mismatch")
|
|
return snapshot
|