312 lines
10 KiB
Python
312 lines
10 KiB
Python
"""Durable filesystem JSON store.
|
|
|
|
Each entity is stored as its own JSON file under a per-type directory. Writes are
|
|
atomic (temp file + os.replace + fsync). Thread-safe via a per-path lock.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from contextlib import contextmanager
|
|
import contextvars
|
|
import fcntl
|
|
import json
|
|
import os
|
|
import threading
|
|
import unicodedata
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Iterator
|
|
|
|
from ..config import Config
|
|
|
|
|
|
class StoreError(Exception):
|
|
pass
|
|
|
|
|
|
class StoreNotFoundError(StoreError):
|
|
"""Raised only when a record does not exist."""
|
|
|
|
|
|
class _Locks:
|
|
def __init__(self) -> None:
|
|
self._locks: dict[str, threading.RLock] = {}
|
|
self._collection_locks: dict[str, threading.RLock] = {}
|
|
self._guard = threading.Lock()
|
|
|
|
def get(self, key: str) -> threading.RLock:
|
|
with self._guard:
|
|
if key not in self._locks:
|
|
self._locks[key] = threading.RLock()
|
|
return self._locks[key]
|
|
|
|
def get_collection(self, key: str) -> threading.RLock:
|
|
with self._guard:
|
|
if key not in self._collection_locks:
|
|
self._collection_locks[key] = threading.RLock()
|
|
return self._collection_locks[key]
|
|
|
|
|
|
_locks = _Locks()
|
|
_held_records: contextvars.ContextVar[frozenset[str]] = contextvars.ContextVar(
|
|
"held_records", default=frozenset()
|
|
)
|
|
|
|
|
|
def new_id(prefix: str) -> str:
|
|
return f"{prefix}-{uuid.uuid4().hex[:12]}"
|
|
|
|
|
|
def contains_control_characters(value: str) -> bool:
|
|
"""Return whether a string contains any Unicode control character."""
|
|
return any(unicodedata.category(char) == "Cc" for char in value)
|
|
|
|
|
|
def _atomic_write(path: Path, value: Any) -> None:
|
|
if not isinstance(value, dict):
|
|
raise StoreError("records must be JSON objects")
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
|
|
try:
|
|
with tmp.open("w", encoding="utf-8") as fh:
|
|
json.dump(value, fh, ensure_ascii=False, indent=2)
|
|
fh.flush()
|
|
os.fsync(fh.fileno())
|
|
os.replace(tmp, path)
|
|
finally:
|
|
if tmp.exists():
|
|
try:
|
|
tmp.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _read_json(path: Path) -> Any:
|
|
try:
|
|
with path.open("r", encoding="utf-8") as fh:
|
|
value = json.load(fh)
|
|
except FileNotFoundError:
|
|
raise
|
|
except (OSError, UnicodeError, ValueError) as exc:
|
|
raise StoreError("record is unreadable") from exc
|
|
if not isinstance(value, dict):
|
|
raise StoreError("record must be a JSON object")
|
|
return value
|
|
|
|
|
|
@contextmanager
|
|
def _exclusive_file_lock(lock_path: Path) -> Iterator[None]:
|
|
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with lock_path.open("a+", encoding="utf-8") as fh:
|
|
fcntl.flock(fh.fileno(), fcntl.LOCK_EX)
|
|
try:
|
|
yield
|
|
finally:
|
|
fcntl.flock(fh.fileno(), fcntl.LOCK_UN)
|
|
|
|
|
|
@contextmanager
|
|
def _process_lock(path: Path) -> Iterator[None]:
|
|
"""Coordinate writes to one record across forked worker processes."""
|
|
with _exclusive_file_lock(path.with_name(f".{path.name}.lock")):
|
|
yield
|
|
|
|
|
|
@contextmanager
|
|
def _maybe_process_lock(path: Path) -> Iterator[None]:
|
|
"""Avoid reacquiring the same process lock inside record_lock()."""
|
|
if str(path) in _held_records.get():
|
|
yield
|
|
else:
|
|
with _process_lock(path):
|
|
yield
|
|
|
|
|
|
@contextmanager
|
|
def _collection_lock(root: Path) -> Iterator[None]:
|
|
"""Coordinate cross-record invariants across threads and worker processes."""
|
|
lock = _locks.get_collection(str(root))
|
|
with lock:
|
|
with _exclusive_file_lock(root / ".collection.lock"):
|
|
yield
|
|
|
|
|
|
class JsonStore:
|
|
"""Simple JSON-file collection with CRUD + locking."""
|
|
|
|
def __init__(self, root: Path, *, key_attr: str = "id") -> None:
|
|
self.root = root
|
|
self.key_attr = key_attr
|
|
self.root.mkdir(parents=True, exist_ok=True)
|
|
|
|
def _path(self, key: str) -> Path:
|
|
if (
|
|
not isinstance(key, str)
|
|
or not key
|
|
or len(key) > 200
|
|
or "/" in key
|
|
or "\\" in key
|
|
or ".." in key
|
|
or key == "."
|
|
or contains_control_characters(key)
|
|
):
|
|
raise StoreError("invalid id")
|
|
try:
|
|
if len(key.encode("utf-8")) > 200:
|
|
raise StoreError("invalid id")
|
|
except UnicodeError as exc:
|
|
raise StoreError("invalid id") from exc
|
|
return self.root / f"{key}.json"
|
|
|
|
def _validate_record_key(self, key: str, value: Any) -> dict[str, Any]:
|
|
if not isinstance(value, dict) or value.get(self.key_attr) != key:
|
|
raise StoreError("record key does not match storage key")
|
|
return value
|
|
|
|
@contextmanager
|
|
def collection_lock(self) -> Iterator[None]:
|
|
"""Hold an exclusive lock for invariants spanning multiple records."""
|
|
with _collection_lock(self.root):
|
|
yield
|
|
|
|
@contextmanager
|
|
def record_lock(self, key: str) -> Iterator[None]:
|
|
"""Hold one record lock across a read/compute/write transaction."""
|
|
path = self._path(key)
|
|
lock = _locks.get(str(path))
|
|
current = _held_records.get()
|
|
already_held = str(path) in current
|
|
with lock:
|
|
process_context = _maybe_process_lock(path) if already_held else _process_lock(path)
|
|
with process_context:
|
|
token = _held_records.set(current | {str(path)})
|
|
try:
|
|
yield
|
|
finally:
|
|
_held_records.reset(token)
|
|
|
|
def create(self, value: dict[str, Any], *, key: str | None = None) -> dict[str, Any]:
|
|
if not isinstance(value, dict):
|
|
raise StoreError("records must be JSON objects")
|
|
requested_key = (
|
|
key
|
|
if key is not None
|
|
else value[self.key_attr]
|
|
if self.key_attr in value
|
|
else new_id(self.key_attr)
|
|
)
|
|
if not isinstance(requested_key, str):
|
|
raise StoreError("invalid id")
|
|
path = self._path(requested_key)
|
|
key = requested_key
|
|
value = dict(value)
|
|
if self.key_attr in value and value[self.key_attr] != key:
|
|
raise StoreError("record key does not match storage key")
|
|
value[self.key_attr] = key
|
|
lock = _locks.get(str(path))
|
|
with lock:
|
|
with _maybe_process_lock(path):
|
|
if path.exists():
|
|
raise StoreError(f"already exists: {key}")
|
|
_atomic_write(path, value)
|
|
return value
|
|
|
|
def get(self, key: str) -> dict[str, Any]:
|
|
path = self._path(key)
|
|
lock = _locks.get(str(path))
|
|
with lock:
|
|
with _maybe_process_lock(path):
|
|
if not path.exists():
|
|
raise StoreNotFoundError(f"not found: {key}")
|
|
try:
|
|
return self._validate_record_key(key, _read_json(path))
|
|
except FileNotFoundError as exc:
|
|
raise StoreNotFoundError(f"not found: {key}") from exc
|
|
|
|
def get_or_none(self, key: str) -> dict[str, Any] | None:
|
|
try:
|
|
return self.get(key)
|
|
except StoreNotFoundError:
|
|
return None
|
|
|
|
def update(self, key: str, **fields: Any) -> dict[str, Any]:
|
|
path = self._path(key)
|
|
lock = _locks.get(str(path))
|
|
with lock:
|
|
with _maybe_process_lock(path):
|
|
if not path.exists():
|
|
raise StoreError(f"not found: {key}")
|
|
cur = _read_json(path)
|
|
self._validate_record_key(key, cur)
|
|
cur.update(fields)
|
|
self._validate_record_key(key, cur)
|
|
_atomic_write(path, cur)
|
|
return cur
|
|
|
|
def update_if(
|
|
self,
|
|
key: str,
|
|
predicate: Callable[[dict[str, Any]], bool],
|
|
**fields: Any,
|
|
) -> dict[str, Any] | None:
|
|
"""Update one record only when ``predicate`` matches its latest value.
|
|
|
|
The predicate check and write share the same per-record lock, allowing
|
|
one-time state transitions to fail closed instead of relying on a
|
|
stale read performed by the caller.
|
|
"""
|
|
path = self._path(key)
|
|
lock = _locks.get(str(path))
|
|
with lock:
|
|
with _maybe_process_lock(path):
|
|
if not path.exists():
|
|
raise StoreError(f"not found: {key}")
|
|
cur = _read_json(path)
|
|
self._validate_record_key(key, cur)
|
|
if not predicate(cur):
|
|
return None
|
|
cur.update(fields)
|
|
self._validate_record_key(key, cur)
|
|
_atomic_write(path, cur)
|
|
return cur
|
|
|
|
def replace(self, key: str, value: dict[str, Any]) -> dict[str, Any]:
|
|
path = self._path(key)
|
|
if not isinstance(value, dict):
|
|
raise StoreError("records must be JSON objects")
|
|
value = self._validate_record_key(key, value)
|
|
lock = _locks.get(str(path))
|
|
with lock:
|
|
with _maybe_process_lock(path):
|
|
_atomic_write(path, value)
|
|
return value
|
|
|
|
def delete(self, key: str) -> None:
|
|
path = self._path(key)
|
|
lock = _locks.get(str(path))
|
|
with lock:
|
|
with _maybe_process_lock(path):
|
|
if path.exists():
|
|
path.unlink()
|
|
|
|
def iter_all(self) -> Iterator[dict[str, Any]]:
|
|
for path in sorted(self.root.glob("*.json")):
|
|
try:
|
|
key = path.name[:-5]
|
|
self._path(key)
|
|
yield self._validate_record_key(key, _read_json(path))
|
|
except FileNotFoundError:
|
|
# A concurrent delete is a normal not-found race; corruption and
|
|
# unreadable records must propagate rather than disappear.
|
|
continue
|
|
|
|
def all(self) -> list[dict[str, Any]]:
|
|
return list(self.iter_all())
|
|
|
|
def iter_where(self, pred: Callable[[dict[str, Any]], bool]) -> Iterator[dict[str, Any]]:
|
|
for row in self.iter_all():
|
|
if pred(row):
|
|
yield row
|
|
|
|
def where(self, pred: Callable[[dict[str, Any]], bool]) -> list[dict[str, Any]]:
|
|
return list(self.iter_where(pred))
|