Files
sales-trainer/backend/app/storage/store.py
T

312 lines
10 KiB
Python

"""Durable filesystem JSON store.
Each entity is stored as its own JSON file under a per-type directory. Writes are
atomic (temp file + os.replace + fsync). Thread-safe via a per-path lock.
"""
from __future__ import annotations
from contextlib import contextmanager
import contextvars
import fcntl
import json
import os
import threading
import unicodedata
import uuid
from pathlib import Path
from typing import Any, Callable, Iterator
from ..config import Config
class StoreError(Exception):
pass
class StoreNotFoundError(StoreError):
"""Raised only when a record does not exist."""
class _Locks:
def __init__(self) -> None:
self._locks: dict[str, threading.RLock] = {}
self._collection_locks: dict[str, threading.RLock] = {}
self._guard = threading.Lock()
def get(self, key: str) -> threading.RLock:
with self._guard:
if key not in self._locks:
self._locks[key] = threading.RLock()
return self._locks[key]
def get_collection(self, key: str) -> threading.RLock:
with self._guard:
if key not in self._collection_locks:
self._collection_locks[key] = threading.RLock()
return self._collection_locks[key]
_locks = _Locks()
_held_records: contextvars.ContextVar[frozenset[str]] = contextvars.ContextVar(
"held_records", default=frozenset()
)
def new_id(prefix: str) -> str:
return f"{prefix}-{uuid.uuid4().hex[:12]}"
def contains_control_characters(value: str) -> bool:
"""Return whether a string contains any Unicode control character."""
return any(unicodedata.category(char) == "Cc" for char in value)
def _atomic_write(path: Path, value: Any) -> None:
if not isinstance(value, dict):
raise StoreError("records must be JSON objects")
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
try:
with tmp.open("w", encoding="utf-8") as fh:
json.dump(value, fh, ensure_ascii=False, indent=2)
fh.flush()
os.fsync(fh.fileno())
os.replace(tmp, path)
finally:
if tmp.exists():
try:
tmp.unlink()
except OSError:
pass
def _read_json(path: Path) -> Any:
try:
with path.open("r", encoding="utf-8") as fh:
value = json.load(fh)
except FileNotFoundError:
raise
except (OSError, UnicodeError, ValueError) as exc:
raise StoreError("record is unreadable") from exc
if not isinstance(value, dict):
raise StoreError("record must be a JSON object")
return value
@contextmanager
def _exclusive_file_lock(lock_path: Path) -> Iterator[None]:
lock_path.parent.mkdir(parents=True, exist_ok=True)
with lock_path.open("a+", encoding="utf-8") as fh:
fcntl.flock(fh.fileno(), fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(fh.fileno(), fcntl.LOCK_UN)
@contextmanager
def _process_lock(path: Path) -> Iterator[None]:
"""Coordinate writes to one record across forked worker processes."""
with _exclusive_file_lock(path.with_name(f".{path.name}.lock")):
yield
@contextmanager
def _maybe_process_lock(path: Path) -> Iterator[None]:
"""Avoid reacquiring the same process lock inside record_lock()."""
if str(path) in _held_records.get():
yield
else:
with _process_lock(path):
yield
@contextmanager
def _collection_lock(root: Path) -> Iterator[None]:
"""Coordinate cross-record invariants across threads and worker processes."""
lock = _locks.get_collection(str(root))
with lock:
with _exclusive_file_lock(root / ".collection.lock"):
yield
class JsonStore:
"""Simple JSON-file collection with CRUD + locking."""
def __init__(self, root: Path, *, key_attr: str = "id") -> None:
self.root = root
self.key_attr = key_attr
self.root.mkdir(parents=True, exist_ok=True)
def _path(self, key: str) -> Path:
if (
not isinstance(key, str)
or not key
or len(key) > 200
or "/" in key
or "\\" in key
or ".." in key
or key == "."
or contains_control_characters(key)
):
raise StoreError("invalid id")
try:
if len(key.encode("utf-8")) > 200:
raise StoreError("invalid id")
except UnicodeError as exc:
raise StoreError("invalid id") from exc
return self.root / f"{key}.json"
def _validate_record_key(self, key: str, value: Any) -> dict[str, Any]:
if not isinstance(value, dict) or value.get(self.key_attr) != key:
raise StoreError("record key does not match storage key")
return value
@contextmanager
def collection_lock(self) -> Iterator[None]:
"""Hold an exclusive lock for invariants spanning multiple records."""
with _collection_lock(self.root):
yield
@contextmanager
def record_lock(self, key: str) -> Iterator[None]:
"""Hold one record lock across a read/compute/write transaction."""
path = self._path(key)
lock = _locks.get(str(path))
current = _held_records.get()
already_held = str(path) in current
with lock:
process_context = _maybe_process_lock(path) if already_held else _process_lock(path)
with process_context:
token = _held_records.set(current | {str(path)})
try:
yield
finally:
_held_records.reset(token)
def create(self, value: dict[str, Any], *, key: str | None = None) -> dict[str, Any]:
if not isinstance(value, dict):
raise StoreError("records must be JSON objects")
requested_key = (
key
if key is not None
else value[self.key_attr]
if self.key_attr in value
else new_id(self.key_attr)
)
if not isinstance(requested_key, str):
raise StoreError("invalid id")
path = self._path(requested_key)
key = requested_key
value = dict(value)
if self.key_attr in value and value[self.key_attr] != key:
raise StoreError("record key does not match storage key")
value[self.key_attr] = key
lock = _locks.get(str(path))
with lock:
with _maybe_process_lock(path):
if path.exists():
raise StoreError(f"already exists: {key}")
_atomic_write(path, value)
return value
def get(self, key: str) -> dict[str, Any]:
path = self._path(key)
lock = _locks.get(str(path))
with lock:
with _maybe_process_lock(path):
if not path.exists():
raise StoreNotFoundError(f"not found: {key}")
try:
return self._validate_record_key(key, _read_json(path))
except FileNotFoundError as exc:
raise StoreNotFoundError(f"not found: {key}") from exc
def get_or_none(self, key: str) -> dict[str, Any] | None:
try:
return self.get(key)
except StoreNotFoundError:
return None
def update(self, key: str, **fields: Any) -> dict[str, Any]:
path = self._path(key)
lock = _locks.get(str(path))
with lock:
with _maybe_process_lock(path):
if not path.exists():
raise StoreError(f"not found: {key}")
cur = _read_json(path)
self._validate_record_key(key, cur)
cur.update(fields)
self._validate_record_key(key, cur)
_atomic_write(path, cur)
return cur
def update_if(
self,
key: str,
predicate: Callable[[dict[str, Any]], bool],
**fields: Any,
) -> dict[str, Any] | None:
"""Update one record only when ``predicate`` matches its latest value.
The predicate check and write share the same per-record lock, allowing
one-time state transitions to fail closed instead of relying on a
stale read performed by the caller.
"""
path = self._path(key)
lock = _locks.get(str(path))
with lock:
with _maybe_process_lock(path):
if not path.exists():
raise StoreError(f"not found: {key}")
cur = _read_json(path)
self._validate_record_key(key, cur)
if not predicate(cur):
return None
cur.update(fields)
self._validate_record_key(key, cur)
_atomic_write(path, cur)
return cur
def replace(self, key: str, value: dict[str, Any]) -> dict[str, Any]:
path = self._path(key)
if not isinstance(value, dict):
raise StoreError("records must be JSON objects")
value = self._validate_record_key(key, value)
lock = _locks.get(str(path))
with lock:
with _maybe_process_lock(path):
_atomic_write(path, value)
return value
def delete(self, key: str) -> None:
path = self._path(key)
lock = _locks.get(str(path))
with lock:
with _maybe_process_lock(path):
if path.exists():
path.unlink()
def iter_all(self) -> Iterator[dict[str, Any]]:
for path in sorted(self.root.glob("*.json")):
try:
key = path.name[:-5]
self._path(key)
yield self._validate_record_key(key, _read_json(path))
except FileNotFoundError:
# A concurrent delete is a normal not-found race; corruption and
# unreadable records must propagate rather than disappear.
continue
def all(self) -> list[dict[str, Any]]:
return list(self.iter_all())
def iter_where(self, pred: Callable[[dict[str, Any]], bool]) -> Iterator[dict[str, Any]]:
for row in self.iter_all():
if pred(row):
yield row
def where(self, pred: Callable[[dict[str, Any]], bool]) -> list[dict[str, Any]]:
return list(self.iter_where(pred))