[verified] Add daily cache layer (fetch-once/day, stale fallback, atomic write)
- DailyCache: JSON file cache keyed by (source/as_of), TTL 24h, atomic tmp+rename write, persists across runs - fetch_or_stale: returns fresh cache, else refetch+cache, else falls back to stale so dashboard is never blanked - 7 tests; full suite pass
This commit is contained in:
94
backend/app/daily_cache.py
Normal file
94
backend/app/daily_cache.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""Simple daily cache for alternative-factor data.
|
||||
|
||||
Per the user's requirement: fetch once per day, keep the raw value keyed by
|
||||
(source, as_of), TTL ~24h, and fall back to the last good cached value when a
|
||||
re-fetch fails (so an interrupted/nightly run doesn't blank the dashboard).
|
||||
|
||||
The cache is a plain JSON file under backend/data/alternative_cache/ — atomic
|
||||
write (tmp + rename) so a crash mid-write can't corrupt it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
_DEFAULT_DIR = Path(__file__).resolve().parent.parent / "data" / "alternative_cache"
|
||||
_DEFAULT_TTL = 24 * 3600 # 24h
|
||||
|
||||
|
||||
class DailyCache:
|
||||
def __init__(self, cache_dir: Path = _DEFAULT_DIR, ttl: float = _DEFAULT_TTL) -> None:
|
||||
self.cache_dir = Path(cache_dir)
|
||||
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.ttl = ttl
|
||||
self._index_path = self.cache_dir / "_index.json"
|
||||
self._index: dict = self._load_index()
|
||||
|
||||
def _load_index(self) -> dict:
|
||||
if self._index_path.exists():
|
||||
try:
|
||||
return json.loads(self._index_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return {}
|
||||
return {}
|
||||
|
||||
def _save_index(self) -> None:
|
||||
tmp = self._index_path.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(self._index, ensure_ascii=False, sort_keys=True),
|
||||
encoding="utf-8")
|
||||
os.replace(tmp, self._index_path)
|
||||
|
||||
def _key_file(self, key: str) -> Path:
|
||||
# key like "tourism/2026-08-25" -> nested path to avoid one huge dir
|
||||
safe = key.replace("/", "__").replace(":", "_")
|
||||
return self.cache_dir / f"{safe}.json"
|
||||
|
||||
def get(self, key: str) -> Optional[dict]:
|
||||
entry = self._index.get(key)
|
||||
if entry is None:
|
||||
return None
|
||||
path = self._key_file(key)
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return None
|
||||
|
||||
def fresh(self, key: str) -> bool:
|
||||
entry = self._index.get(key)
|
||||
if entry is None:
|
||||
return False
|
||||
return (time.time() - entry.get("ts", 0)) <= self.ttl
|
||||
|
||||
def set(self, key: str, value: dict) -> None:
|
||||
path = self._key_file(key)
|
||||
tmp = path.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(value, ensure_ascii=False, sort_keys=True),
|
||||
encoding="utf-8")
|
||||
os.replace(tmp, path)
|
||||
self._index[key] = {"ts": time.time()}
|
||||
self._save_index()
|
||||
|
||||
# Convenience: fetch-or-fallback
|
||||
def fetch_or_stale(self, key: str, fetcher, *args, **kwargs) -> dict:
|
||||
"""Return fresh cached value, else call fetcher and cache it, else stale."""
|
||||
cached = self.get(key)
|
||||
if cached is not None and self.fresh(key):
|
||||
return cached
|
||||
try:
|
||||
value = fetcher(*args, **kwargs)
|
||||
self.set(key, value)
|
||||
return value
|
||||
except Exception:
|
||||
# fall back to stale cache so the dashboard is never blanked
|
||||
if cached is not None:
|
||||
cached = dict(cached)
|
||||
cached["_stale"] = True
|
||||
return cached
|
||||
raise
|
||||
Reference in New Issue
Block a user