[verified] bind snapshots to manifests and raw hashes
This commit is contained in:
@@ -162,12 +162,18 @@ def create_app(config: dict[str, Any] | None = None) -> Flask:
|
|||||||
|
|
||||||
def _price_health_payload() -> dict[str, Any]:
|
def _price_health_payload() -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
snapshots = list(app.extensions["price_store"].load_manifest().get("snapshots", {}).values())
|
price_store = app.extensions["price_store"]
|
||||||
|
snapshots = list(price_store.load_manifest().get("snapshots", {}).values())
|
||||||
except PriceSourceError as exc:
|
except PriceSourceError as exc:
|
||||||
return {"available": False, "status": "error", "error": str(exc), "snapshot_count": 0}
|
return {"available": False, "status": "error", "error": str(exc), "snapshot_count": 0}
|
||||||
if not snapshots:
|
if not snapshots:
|
||||||
return {"available": False, "status": "missing", "point_in_time": False, "snapshot_count": 0}
|
return {"available": False, "status": "missing", "point_in_time": False, "snapshot_count": 0}
|
||||||
latest = max(snapshots, key=lambda item: str(item.get("retrieved_at", "")))
|
latest = max(snapshots, key=lambda item: str(item.get("retrieved_at", "")))
|
||||||
|
try:
|
||||||
|
snapshot = price_store.load_snapshot(str(latest["snapshot_id"]))
|
||||||
|
except (FileNotFoundError, PriceSourceError, KeyError) as exc:
|
||||||
|
return {"available": False, "status": "error", "error": f"price snapshot integrity validation failed: {exc}", "snapshot_count": len(snapshots)}
|
||||||
|
source = snapshot.get("source", {})
|
||||||
return {
|
return {
|
||||||
"available": True,
|
"available": True,
|
||||||
"status": "available",
|
"status": "available",
|
||||||
@@ -176,8 +182,8 @@ def create_app(config: dict[str, Any] | None = None) -> Flask:
|
|||||||
"retrieved_at": latest.get("retrieved_at"),
|
"retrieved_at": latest.get("retrieved_at"),
|
||||||
"period_start": latest.get("period_start"),
|
"period_start": latest.get("period_start"),
|
||||||
"period_end": latest.get("period_end"),
|
"period_end": latest.get("period_end"),
|
||||||
"quality": latest.get("quality"),
|
"quality": source.get("quality"),
|
||||||
"point_in_time": bool(latest.get("point_in_time")),
|
"point_in_time": source.get("point_in_time"),
|
||||||
"symbols": latest.get("symbols", []),
|
"symbols": latest.get("symbols", []),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ DEFAULT_SYMBOL_MAP = {
|
|||||||
}
|
}
|
||||||
PRICE_SCHEMA_VERSION = 1
|
PRICE_SCHEMA_VERSION = 1
|
||||||
PRICE_PARSER_VERSION = "yahoo-chart-v1"
|
PRICE_PARSER_VERSION = "yahoo-chart-v1"
|
||||||
|
NORMALIZED_HASH_ALGORITHM = "sha256-json-canonical-v1"
|
||||||
MAX_PAYLOAD_BYTES = 5_000_000
|
MAX_PAYLOAD_BYTES = 5_000_000
|
||||||
_PROVIDER_SYMBOL_RE = re.compile(r"^[A-Za-z0-9^._-]{1,64}$")
|
_PROVIDER_SYMBOL_RE = re.compile(r"^[A-Za-z0-9^._-]{1,64}$")
|
||||||
|
|
||||||
@@ -57,6 +58,15 @@ def _combined_hash(raw_payloads: Mapping[str, bytes]) -> str:
|
|||||||
return digest.hexdigest()
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _normalized_snapshot_hash(snapshot: dict[str, Any]) -> str:
|
||||||
|
payload = copy.deepcopy(snapshot)
|
||||||
|
source = payload.get("source")
|
||||||
|
if isinstance(source, dict):
|
||||||
|
source.pop("normalized_snapshot_hash", None)
|
||||||
|
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
return hashlib.sha256(encoded).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
def _raw_filename(provider_symbol: str) -> str:
|
def _raw_filename(provider_symbol: str) -> str:
|
||||||
if not _PROVIDER_SYMBOL_RE.fullmatch(provider_symbol):
|
if not _PROVIDER_SYMBOL_RE.fullmatch(provider_symbol):
|
||||||
raise PriceSourceError("invalid provider symbol")
|
raise PriceSourceError("invalid provider symbol")
|
||||||
@@ -249,6 +259,11 @@ class PriceSnapshotStore:
|
|||||||
source = snapshot.get("source")
|
source = snapshot.get("source")
|
||||||
if not isinstance(source, dict) or source.get("snapshot_id") != snapshot_id:
|
if not isinstance(source, dict) or source.get("snapshot_id") != snapshot_id:
|
||||||
raise PriceSourceError("price snapshot identity mismatch")
|
raise PriceSourceError("price snapshot identity mismatch")
|
||||||
|
if not isinstance(source.get("point_in_time"), bool):
|
||||||
|
raise PriceSourceError("price snapshot metadata point_in_time must be boolean")
|
||||||
|
declared_snapshot_hash = source.get("normalized_snapshot_hash")
|
||||||
|
if source.get("normalized_hash_algorithm") != NORMALIZED_HASH_ALGORITHM or not isinstance(declared_snapshot_hash, str) or _normalized_snapshot_hash(snapshot) != declared_snapshot_hash:
|
||||||
|
raise PriceSourceError("price normalized snapshot hash mismatch")
|
||||||
raw_files = source.get("raw_payload_files")
|
raw_files = source.get("raw_payload_files")
|
||||||
if not isinstance(raw_files, dict) or not raw_files:
|
if not isinstance(raw_files, dict) or not raw_files:
|
||||||
raise PriceSourceError("price snapshot raw payload manifest is missing")
|
raise PriceSourceError("price snapshot raw payload manifest is missing")
|
||||||
@@ -267,7 +282,13 @@ class PriceSnapshotStore:
|
|||||||
if actual_hash != source.get("raw_payload_hash"):
|
if actual_hash != source.get("raw_payload_hash"):
|
||||||
raise PriceSourceError("price raw payload hash mismatch")
|
raise PriceSourceError("price raw payload hash mismatch")
|
||||||
manifest_entry = self.load_manifest().get("snapshots", {}).get(snapshot_id)
|
manifest_entry = self.load_manifest().get("snapshots", {}).get(snapshot_id)
|
||||||
if isinstance(manifest_entry, dict) and manifest_entry.get("raw_payload_hash") != actual_hash:
|
if not isinstance(manifest_entry, dict):
|
||||||
|
raise PriceSourceError("price manifest entry is missing")
|
||||||
|
if manifest_entry.get("source_id") != source.get("source_id") or manifest_entry.get("quality") != source.get("quality"):
|
||||||
|
raise PriceSourceError("price manifest metadata mismatch")
|
||||||
|
if manifest_entry.get("point_in_time") is not source.get("point_in_time"):
|
||||||
|
raise PriceSourceError("price manifest metadata mismatch")
|
||||||
|
if manifest_entry.get("raw_payload_hash") != actual_hash or manifest_entry.get("normalized_snapshot_hash") != declared_snapshot_hash or manifest_entry.get("normalized_hash_algorithm") != NORMALIZED_HASH_ALGORITHM:
|
||||||
raise PriceSourceError("price manifest hash mismatch")
|
raise PriceSourceError("price manifest hash mismatch")
|
||||||
return snapshot
|
return snapshot
|
||||||
|
|
||||||
@@ -288,6 +309,8 @@ class PriceSnapshotStore:
|
|||||||
raw_hash = _combined_hash(raw_payloads)
|
raw_hash = _combined_hash(raw_payloads)
|
||||||
if source.get("raw_payload_hash") and source["raw_payload_hash"] != raw_hash:
|
if source.get("raw_payload_hash") and source["raw_payload_hash"] != raw_hash:
|
||||||
raise PriceSourceError("raw payload hash does not match price snapshot")
|
raise PriceSourceError("raw payload hash does not match price snapshot")
|
||||||
|
if "point_in_time" in source and not isinstance(source["point_in_time"], bool):
|
||||||
|
raise PriceSourceError("price snapshot metadata point_in_time must be boolean")
|
||||||
stored = copy.deepcopy(snapshot)
|
stored = copy.deepcopy(snapshot)
|
||||||
stored_source = stored["source"]
|
stored_source = stored["source"]
|
||||||
snapshot_id = str(stored_source["snapshot_id"])
|
snapshot_id = str(stored_source["snapshot_id"])
|
||||||
@@ -296,6 +319,8 @@ class PriceSnapshotStore:
|
|||||||
stored_source["raw_payload_hash"] = raw_hash
|
stored_source["raw_payload_hash"] = raw_hash
|
||||||
stored_source["raw_payload_files"] = {symbol: _raw_filename(symbol) for symbol in raw_payloads}
|
stored_source["raw_payload_files"] = {symbol: _raw_filename(symbol) for symbol in raw_payloads}
|
||||||
stored_source["snapshot_file"] = f"{snapshot_id}.json"
|
stored_source["snapshot_file"] = f"{snapshot_id}.json"
|
||||||
|
stored_source["normalized_hash_algorithm"] = NORMALIZED_HASH_ALGORITHM
|
||||||
|
stored_source["normalized_snapshot_hash"] = _normalized_snapshot_hash(stored)
|
||||||
for provider_symbol, payload in raw_payloads.items():
|
for provider_symbol, payload in raw_payloads.items():
|
||||||
_atomic_write(self.raw_dir / snapshot_id / _raw_filename(provider_symbol), payload)
|
_atomic_write(self.raw_dir / snapshot_id / _raw_filename(provider_symbol), payload)
|
||||||
_atomic_write(self.snapshot_dir / stored_source["snapshot_file"], (json.dumps(stored, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8"))
|
_atomic_write(self.snapshot_dir / stored_source["snapshot_file"], (json.dumps(stored, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8"))
|
||||||
@@ -307,6 +332,8 @@ class PriceSnapshotStore:
|
|||||||
"period_start": stored_source.get("period_start"),
|
"period_start": stored_source.get("period_start"),
|
||||||
"period_end": stored_source.get("period_end"),
|
"period_end": stored_source.get("period_end"),
|
||||||
"raw_payload_hash": raw_hash,
|
"raw_payload_hash": raw_hash,
|
||||||
|
"normalized_snapshot_hash": stored_source["normalized_snapshot_hash"],
|
||||||
|
"normalized_hash_algorithm": NORMALIZED_HASH_ALGORITHM,
|
||||||
"parser_version": stored_source.get("parser_version"),
|
"parser_version": stored_source.get("parser_version"),
|
||||||
"quality": stored_source.get("quality"),
|
"quality": stored_source.get("quality"),
|
||||||
"point_in_time": stored_source.get("point_in_time"),
|
"point_in_time": stored_source.get("point_in_time"),
|
||||||
|
|||||||
@@ -133,7 +133,7 @@ def _validate_config(windows: Sequence[int], cost_bps: float, min_events: int, e
|
|||||||
def _price_gate(price_entry: Mapping[str, Any] | None) -> dict[str, Any]:
|
def _price_gate(price_entry: Mapping[str, Any] | None) -> dict[str, Any]:
|
||||||
if not price_entry:
|
if not price_entry:
|
||||||
return {"status": "blocked", "reason": "price_snapshot_missing", "snapshot_count": 0}
|
return {"status": "blocked", "reason": "price_snapshot_missing", "snapshot_count": 0}
|
||||||
if not bool(price_entry.get("point_in_time")):
|
if price_entry.get("point_in_time") is not True:
|
||||||
return {
|
return {
|
||||||
"status": "blocked",
|
"status": "blocked",
|
||||||
"reason": "price_series_not_point_in_time",
|
"reason": "price_series_not_point_in_time",
|
||||||
@@ -200,7 +200,28 @@ def run_tourism_research(
|
|||||||
except (VintageStoreError, PriceSourceError) as exc:
|
except (VintageStoreError, PriceSourceError) as exc:
|
||||||
raise ResearchRunError(str(exc)) from exc
|
raise ResearchRunError(str(exc)) from exc
|
||||||
vintage_gate = assess_backtest_readiness(vintages, min_events=min_events)
|
vintage_gate = assess_backtest_readiness(vintages, min_events=min_events)
|
||||||
price_gate = _price_gate(price_entry)
|
validated_vintage_snapshots: dict[str, dict[str, Any]] = {}
|
||||||
|
try:
|
||||||
|
for entry in vintages:
|
||||||
|
vintage_id = str(entry["vintage_id"])
|
||||||
|
validated_vintage_snapshots[vintage_id] = vintage_store.load_snapshot(vintage_id)
|
||||||
|
except (FileNotFoundError, VintageStoreError, KeyError) as exc:
|
||||||
|
raise ResearchRunError(f"vintage snapshot integrity validation failed: {exc}") from exc
|
||||||
|
price_snapshot: dict[str, Any] | None = None
|
||||||
|
price_gate_entry = price_entry
|
||||||
|
if price_entry:
|
||||||
|
try:
|
||||||
|
price_snapshot = price_store.load_snapshot(str(price_entry["snapshot_id"]))
|
||||||
|
except (FileNotFoundError, PriceSourceError, KeyError) as exc:
|
||||||
|
raise ResearchRunError(f"price snapshot integrity validation failed: {exc}") from exc
|
||||||
|
price_source = price_snapshot.get("source", {})
|
||||||
|
price_gate_entry = {
|
||||||
|
**price_entry,
|
||||||
|
"point_in_time": price_source.get("point_in_time"),
|
||||||
|
"quality": price_source.get("quality"),
|
||||||
|
"normalized_snapshot_hash": price_source.get("normalized_snapshot_hash"),
|
||||||
|
}
|
||||||
|
price_gate = _price_gate(price_gate_entry)
|
||||||
input_vintages = [
|
input_vintages = [
|
||||||
{
|
{
|
||||||
"vintage_id": entry.get("vintage_id"),
|
"vintage_id": entry.get("vintage_id"),
|
||||||
@@ -213,7 +234,7 @@ def run_tourism_research(
|
|||||||
}
|
}
|
||||||
for entry in vintages
|
for entry in vintages
|
||||||
]
|
]
|
||||||
input_price = copy.deepcopy(price_entry) if price_entry else None
|
input_price = copy.deepcopy(price_gate_entry) if price_gate_entry else None
|
||||||
config = {
|
config = {
|
||||||
"min_events": min_events,
|
"min_events": min_events,
|
||||||
"windows": windows,
|
"windows": windows,
|
||||||
@@ -244,14 +265,8 @@ def run_tourism_research(
|
|||||||
if price_gate["status"] != "ready":
|
if price_gate["status"] != "ready":
|
||||||
report["reason"] = price_gate["reason"]
|
report["reason"] = price_gate["reason"]
|
||||||
return run_store.persist(report)
|
return run_store.persist(report)
|
||||||
if not price_entry:
|
if not price_entry or price_snapshot is None:
|
||||||
raise ResearchRunError("price snapshot disappeared after readiness check")
|
raise ResearchRunError("price snapshot disappeared after readiness check")
|
||||||
try:
|
|
||||||
price_snapshot = price_store.load_snapshot(str(price_entry["snapshot_id"]))
|
|
||||||
except (FileNotFoundError, PriceSourceError) as exc:
|
|
||||||
report["reason"] = "price_snapshot_unreadable"
|
|
||||||
report["gates"]["prices"] = {**price_gate, "status": "blocked", "reason": report["reason"], "error": str(exc)}
|
|
||||||
return run_store.persist(report)
|
|
||||||
price_source = price_snapshot.get("source", {})
|
price_source = price_snapshot.get("source", {})
|
||||||
price_field = str(price_source.get("return_price_field", "close"))
|
price_field = str(price_source.get("return_price_field", "close"))
|
||||||
series = price_snapshot.get("series")
|
series = price_snapshot.get("series")
|
||||||
@@ -261,7 +276,7 @@ def run_tourism_research(
|
|||||||
events: list[dict[str, Any]] = []
|
events: list[dict[str, Any]] = []
|
||||||
try:
|
try:
|
||||||
for entry in vintages:
|
for entry in vintages:
|
||||||
snapshot = vintage_store.load_snapshot(str(entry["vintage_id"]))
|
snapshot = validated_vintage_snapshots[str(entry["vintage_id"])]
|
||||||
result = compute_tourism_signal(snapshot)
|
result = compute_tourism_signal(snapshot)
|
||||||
events.append({
|
events.append({
|
||||||
"event_id": entry["vintage_id"],
|
"event_id": entry["vintage_id"],
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ from pathlib import Path
|
|||||||
from typing import Any, Iterable
|
from typing import Any, Iterable
|
||||||
|
|
||||||
MANIFEST_SCHEMA_VERSION = 1
|
MANIFEST_SCHEMA_VERSION = 1
|
||||||
|
NORMALIZED_HASH_ALGORITHM = "sha256-json-canonical-v1"
|
||||||
_VINTAGE_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}")
|
_VINTAGE_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}")
|
||||||
_FILENAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,191}\.(?:html|json)")
|
_FILENAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,191}\.(?:html|json)")
|
||||||
|
|
||||||
@@ -29,6 +30,15 @@ def _parse_timestamp(value: str) -> datetime:
|
|||||||
return parsed
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
|
def _normalized_snapshot_hash(snapshot: dict[str, Any]) -> str:
|
||||||
|
payload = copy.deepcopy(snapshot)
|
||||||
|
source = payload.get("source")
|
||||||
|
if isinstance(source, dict):
|
||||||
|
source.pop("normalized_snapshot_hash", None)
|
||||||
|
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
return hashlib.sha256(encoded).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
def _safe_vintage_id(vintage_id: str) -> str:
|
def _safe_vintage_id(vintage_id: str) -> str:
|
||||||
if not _VINTAGE_ID_RE.fullmatch(vintage_id):
|
if not _VINTAGE_ID_RE.fullmatch(vintage_id):
|
||||||
raise VintageStoreError("invalid vintage_id")
|
raise VintageStoreError("invalid vintage_id")
|
||||||
@@ -122,7 +132,9 @@ class VintageStore:
|
|||||||
)
|
)
|
||||||
revision_status = existing.get("revision_status") if existing else ("revised" if same_release else "initial")
|
revision_status = existing.get("revision_status") if existing else ("revised" if same_release else "initial")
|
||||||
source["revision_status"] = revision_status
|
source["revision_status"] = revision_status
|
||||||
|
source["normalized_hash_algorithm"] = NORMALIZED_HASH_ALGORITHM
|
||||||
stored_snapshot["source"] = source
|
stored_snapshot["source"] = source
|
||||||
|
source["normalized_snapshot_hash"] = _normalized_snapshot_hash(stored_snapshot)
|
||||||
|
|
||||||
_atomic_write_bytes(self.raw_dir / source["raw_snapshot_file"], raw_bytes)
|
_atomic_write_bytes(self.raw_dir / source["raw_snapshot_file"], raw_bytes)
|
||||||
_atomic_write_json(self.snapshot_dir / source["snapshot_file"], stored_snapshot)
|
_atomic_write_json(self.snapshot_dir / source["snapshot_file"], stored_snapshot)
|
||||||
@@ -138,6 +150,8 @@ class VintageStore:
|
|||||||
"release_status": source.get("release_status"),
|
"release_status": source.get("release_status"),
|
||||||
"revision_status": revision_status,
|
"revision_status": revision_status,
|
||||||
"raw_payload_hash": raw_hash,
|
"raw_payload_hash": raw_hash,
|
||||||
|
"normalized_snapshot_hash": source["normalized_snapshot_hash"],
|
||||||
|
"normalized_hash_algorithm": NORMALIZED_HASH_ALGORITHM,
|
||||||
"parser_version": source.get("parser_version"),
|
"parser_version": source.get("parser_version"),
|
||||||
"raw_snapshot_file": source["raw_snapshot_file"],
|
"raw_snapshot_file": source["raw_snapshot_file"],
|
||||||
"snapshot_file": source["snapshot_file"],
|
"snapshot_file": source["snapshot_file"],
|
||||||
@@ -162,6 +176,25 @@ class VintageStore:
|
|||||||
snapshot = json.loads(snapshot_path.read_text(encoding="utf-8"))
|
snapshot = json.loads(snapshot_path.read_text(encoding="utf-8"))
|
||||||
except (OSError, json.JSONDecodeError) as exc:
|
except (OSError, json.JSONDecodeError) as exc:
|
||||||
raise VintageStoreError("vintage snapshot is unreadable") from exc
|
raise VintageStoreError("vintage snapshot is unreadable") from exc
|
||||||
if snapshot.get("source", {}).get("vintage_id") != vintage_id:
|
source = snapshot.get("source")
|
||||||
|
if not isinstance(source, dict) or source.get("vintage_id") != vintage_id:
|
||||||
raise VintageStoreError("vintage snapshot identity mismatch")
|
raise VintageStoreError("vintage snapshot identity mismatch")
|
||||||
|
declared_snapshot_hash = source.get("normalized_snapshot_hash")
|
||||||
|
if source.get("normalized_hash_algorithm") != NORMALIZED_HASH_ALGORITHM or not isinstance(declared_snapshot_hash, str) or _normalized_snapshot_hash(snapshot) != declared_snapshot_hash:
|
||||||
|
raise VintageStoreError("vintage normalized snapshot hash mismatch")
|
||||||
|
raw_filename = _safe_filename(str(source.get("raw_snapshot_file", "")), ".html")
|
||||||
|
raw_path = (self.raw_dir / raw_filename).resolve()
|
||||||
|
if raw_path.parent != self.raw_dir.resolve() or not raw_path.is_file():
|
||||||
|
raise VintageStoreError("vintage raw payload is missing")
|
||||||
|
if hashlib.sha256(raw_path.read_bytes()).hexdigest() != source.get("raw_payload_hash"):
|
||||||
|
raise VintageStoreError("vintage raw payload hash mismatch")
|
||||||
|
manifest_entry = self.load_manifest().get("vintages", {}).get(vintage_id)
|
||||||
|
if not isinstance(manifest_entry, dict):
|
||||||
|
raise VintageStoreError("vintage manifest entry is missing")
|
||||||
|
if manifest_entry.get("source_id") != source.get("source_id"):
|
||||||
|
raise VintageStoreError("vintage manifest metadata mismatch")
|
||||||
|
if _parse_timestamp(str(manifest_entry.get("published_at"))) != _parse_timestamp(str(source.get("published_at"))):
|
||||||
|
raise VintageStoreError("vintage manifest metadata mismatch")
|
||||||
|
if manifest_entry.get("raw_payload_hash") != source.get("raw_payload_hash") or manifest_entry.get("normalized_snapshot_hash") != declared_snapshot_hash or manifest_entry.get("normalized_hash_algorithm") != NORMALIZED_HASH_ALGORITHM:
|
||||||
|
raise VintageStoreError("vintage manifest hash mismatch")
|
||||||
return snapshot
|
return snapshot
|
||||||
|
|||||||
@@ -4,8 +4,8 @@ import tempfile
|
|||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from app.prices import PriceSnapshotStore
|
from app.prices import PriceSnapshotStore, PriceSourceError
|
||||||
from app.research import ResearchRunStore, run_tourism_research
|
from app.research import ResearchRunError, ResearchRunStore, run_tourism_research
|
||||||
from app.vintages import VintageStore
|
from app.vintages import VintageStore
|
||||||
|
|
||||||
|
|
||||||
@@ -69,6 +69,19 @@ class ResearchRunTests(unittest.TestCase):
|
|||||||
self.assertEqual(loaded["source"]["snapshot_id"], "prices-test")
|
self.assertEqual(loaded["source"]["snapshot_id"], "prices-test")
|
||||||
self.assertEqual(loaded["source"]["raw_payload_hash"], store.load_manifest()["snapshots"]["prices-test"]["raw_payload_hash"])
|
self.assertEqual(loaded["source"]["raw_payload_hash"], store.load_manifest()["snapshots"]["prices-test"]["raw_payload_hash"])
|
||||||
|
|
||||||
|
def test_price_store_rejects_manifest_metadata_tampering(self):
|
||||||
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
|
root = Path(temp_dir)
|
||||||
|
store = PriceSnapshotStore(root)
|
||||||
|
snapshot, raw = price_snapshot(point_in_time=False)
|
||||||
|
store.persist(snapshot, raw)
|
||||||
|
manifest_path = root / "manifest.json"
|
||||||
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
manifest["snapshots"]["prices-test"]["point_in_time"] = True
|
||||||
|
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||||
|
with self.assertRaisesRegex(PriceSourceError, "metadata"):
|
||||||
|
store.load_snapshot("prices-test")
|
||||||
|
|
||||||
def test_runner_persists_blocked_report_when_vintage_count_is_insufficient(self):
|
def test_runner_persists_blocked_report_when_vintage_count_is_insufficient(self):
|
||||||
with tempfile.TemporaryDirectory() as temp_dir:
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
root = Path(temp_dir)
|
root = Path(temp_dir)
|
||||||
@@ -125,6 +138,25 @@ class ResearchRunTests(unittest.TestCase):
|
|||||||
self.assertEqual(len(report["inputs"]["vintages"]), 2)
|
self.assertEqual(len(report["inputs"]["vintages"]), 2)
|
||||||
self.assertIn("v1-revised", {entry["vintage_id"] for entry in report["inputs"]["vintages"]})
|
self.assertIn("v1-revised", {entry["vintage_id"] for entry in report["inputs"]["vintages"]})
|
||||||
|
|
||||||
|
def test_runner_does_not_replay_cached_ready_report_after_price_tampering(self):
|
||||||
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
|
root = Path(temp_dir)
|
||||||
|
vintage_store = VintageStore(root / "tourism")
|
||||||
|
for index, published_at in ((1, "2026-01-01T08:00:00+07:00"), (2, "2026-01-02T08:00:00+07:00")):
|
||||||
|
vintage_store.persist(f"tourism-{index}".encode(), tourism_snapshot(f"v{index}", published_at))
|
||||||
|
price_store = PriceSnapshotStore(root / "prices")
|
||||||
|
snapshot, raw = price_snapshot(point_in_time=True)
|
||||||
|
price_store.persist(snapshot, raw)
|
||||||
|
run_store = ResearchRunStore(root / "runs")
|
||||||
|
report = run_tourism_research(vintage_store, price_store, run_store, min_events=2, windows=(1,))
|
||||||
|
self.assertEqual(report["status"], "ready")
|
||||||
|
manifest_path = root / "prices" / "manifest.json"
|
||||||
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
manifest["snapshots"]["prices-test"]["point_in_time"] = False
|
||||||
|
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||||
|
with self.assertRaisesRegex(ResearchRunError, "integrity"):
|
||||||
|
run_tourism_research(vintage_store, price_store, run_store, min_events=2, windows=(1,))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -115,6 +115,19 @@ class VintageStoreTests(unittest.TestCase):
|
|||||||
with self.assertRaises(ValueError):
|
with self.assertRaises(ValueError):
|
||||||
store.load_snapshot("../secret")
|
store.load_snapshot("../secret")
|
||||||
|
|
||||||
|
def test_snapshot_load_rejects_tampered_normalized_payload(self):
|
||||||
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
|
root = Path(temp_dir)
|
||||||
|
store = VintageStore(root)
|
||||||
|
snapshot = sample_snapshot()
|
||||||
|
store.persist(REPORT_HTML.encode("utf-8"), snapshot)
|
||||||
|
snapshot_path = root / "snapshots" / f"{snapshot['source']['vintage_id']}.json"
|
||||||
|
payload = json.loads(snapshot_path.read_text(encoding="utf-8"))
|
||||||
|
payload["observations"][0]["value"] = 999999
|
||||||
|
snapshot_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||||
|
with self.assertRaisesRegex(ValueError, "hash"):
|
||||||
|
store.load_snapshot(snapshot["source"]["vintage_id"])
|
||||||
|
|
||||||
def test_collector_returns_manifested_snapshot_summary(self):
|
def test_collector_returns_manifested_snapshot_summary(self):
|
||||||
with tempfile.TemporaryDirectory() as temp_dir:
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
result = collect_bot_vintage(
|
result = collect_bot_vintage(
|
||||||
|
|||||||
@@ -4,8 +4,8 @@
|
|||||||
|
|
||||||
- Path: `/Users/kunthawat/Gitea/set50-alternative-data-platform`
|
- Path: `/Users/kunthawat/Gitea/set50-alternative-data-platform`
|
||||||
- Branch: `main`
|
- Branch: `main`
|
||||||
- Verified code commit: `7f7a614` — `[verified] add SET price snapshot adapter` (M2.5 pending commit)
|
- Verified code commit: `f55ff69` — `[verified] add frozen research runner and durable paper ledger` (integrity hardening pending commit)
|
||||||
- Current milestone: M2.5 research runner and durable paper workflow complete; validated backtest blocked
|
- Current milestone: M2.5 integrity hardening complete; validated backtest blocked
|
||||||
- Mode: research + paper only
|
- Mode: research + paper only
|
||||||
- Frontend: Vue 3 + Vite
|
- Frontend: Vue 3 + Vite
|
||||||
- Backend: Flask `0.5.0`
|
- Backend: Flask `0.5.0`
|
||||||
@@ -29,6 +29,7 @@
|
|||||||
- Frozen research runner: `POST /api/v1/research/tourism/run` and `GET /api/v1/research/tourism/latest`.
|
- Frozen research runner: `POST /api/v1/research/tourism/run` and `GET /api/v1/research/tourism/latest`.
|
||||||
- Headless runner: `backend/scripts/run_tourism_research.py`.
|
- Headless runner: `backend/scripts/run_tourism_research.py`.
|
||||||
- Durable atomic paper ledger under `backend/data/paper/ledger.json` when configured.
|
- Durable atomic paper ledger under `backend/data/paper/ledger.json` when configured.
|
||||||
|
- Raw and normalized snapshot integrity binding with explicit `sha256-json-canonical-v1` metadata.
|
||||||
- Ranked target weights and LONG/SHORT/NEUTRAL classification.
|
- Ranked target weights and LONG/SHORT/NEUTRAL classification.
|
||||||
- English dashboard with live/provisional source label, sign-aware surprise copy and lineage fields.
|
- English dashboard with live/provisional source label, sign-aware surprise copy and lineage fields.
|
||||||
- HttpOnly paper session and internal paper ledger.
|
- HttpOnly paper session and internal paper ledger.
|
||||||
@@ -98,10 +99,12 @@ Paper writes use a server-side token exchange and HttpOnly `paper_session` cooki
|
|||||||
- One independent source release is not enough for a valid event study; current historical rows are not treated as point-in-time vintages.
|
- One independent source release is not enough for a valid event study; current historical rows are not treated as point-in-time vintages.
|
||||||
- The event-study engine is deterministic and tested. Yahoo price history is connected for plumbing, but it is revised vendor history, not point-in-time data.
|
- The event-study engine is deterministic and tested. Yahoo price history is connected for plumbing, but it is revised vendor history, not point-in-time data.
|
||||||
- Research runner is operational and replayable, but correctly emits a blocked report until both evidence gates pass.
|
- Research runner is operational and replayable, but correctly emits a blocked report until both evidence gates pass.
|
||||||
|
- Snapshot loads fail closed when raw files, normalized payloads or manifest metadata do not match their recorded hashes.
|
||||||
|
- The hash boundary protects local artifacts against corruption/partial writes; hostile host-level rewrite of code, manifests and runtime environment is outside this local threat model.
|
||||||
- Revisions are deduplicated by canonical `(source_id, published_at)`; the latest observed revision is selected once.
|
- Revisions are deduplicated by canonical `(source_id, published_at)`; the latest observed revision is selected once.
|
||||||
- Event-study default execution is the next trading session, including weekend/holiday event dates.
|
- Event-study default execution is the next trading session, including weekend/holiday event dates.
|
||||||
- Browser screenshot verification remains blocked by the Chrome remote-debugging permission prompt; served HTML/source, live API, fresh Vite build and replay integrity were verified instead.
|
- Browser screenshot verification remains blocked by the Chrome remote-debugging permission prompt; served HTML/source, live API, fresh Vite build and replay integrity were verified instead.
|
||||||
|
|
||||||
## Exact next action
|
## Exact next action
|
||||||
|
|
||||||
Collect independent BOT releases over time and replace/supplement revised vendor history with a point-in-time daily price source. The operator can run the frozen research check now; only raise the gate when both requirements pass. Add another metric only when its historical release coverage is real.
|
Collect independent BOT releases over time and replace/supplement revised vendor history with a point-in-time daily price source. The operator can run the frozen research check now; only raise the gate when both requirements pass. Add a trusted deployment signing key before treating the app as a multi-user or hostile-host service.
|
||||||
|
|||||||
@@ -10,6 +10,7 @@
|
|||||||
| M2.3 event-study gate | complete/blocked | 30 tests, pure engine and truthful 409 readiness API | add point-in-time price provider |
|
| M2.3 event-study gate | complete/blocked | 30 tests, pure engine and truthful 409 readiness API | add point-in-time price provider |
|
||||||
| M2.4 price snapshot adapter | complete/blocked | 38 tests, live 9-symbol Yahoo snapshot, revised-history gate | evaluate point-in-time price source |
|
| M2.4 price snapshot adapter | complete/blocked | 38 tests, live 9-symbol Yahoo snapshot, revised-history gate | evaluate point-in-time price source |
|
||||||
| M2.5 research runner + durable paper ledger | complete/blocked | 54 tests, immutable blocked report, restart-safe paper path, live API/UI workflow | collect independent releases and point-in-time prices |
|
| M2.5 research runner + durable paper ledger | complete/blocked | 54 tests, immutable blocked report, restart-safe paper path, live API/UI workflow | collect independent releases and point-in-time prices |
|
||||||
|
| M2.5 integrity hardening | complete/blocked | 57 tests, normalized/raw hash binding, manifest cross-checks, live re-collection | protect point-in-time gate with a trusted deployment secret if threat model expands |
|
||||||
| Tourism deterministic signal | complete | live foreign-arrivals YoY surprise | add occupancy/airport metric |
|
| Tourism deterministic signal | complete | live foreign-arrivals YoY surprise | add occupancy/airport metric |
|
||||||
| Internal paper ledger | complete | atomic local JSON persistence and restart test | shared store before multi-worker deployment |
|
| Internal paper ledger | complete | atomic local JSON persistence and restart test | shared store before multi-worker deployment |
|
||||||
| Dashboard | complete | Vite build + served source check with live-sign copy | visual browser capture after permission is available |
|
| Dashboard | complete | Vite build + served source check with live-sign copy | visual browser capture after permission is available |
|
||||||
@@ -27,7 +28,7 @@
|
|||||||
|
|
||||||
## Verification
|
## Verification
|
||||||
|
|
||||||
- Backend: 54 unittest tests pass.
|
- Backend: 57 unittest tests pass.
|
||||||
- Independent M1 review: **PASSED**; no concrete security or logic blockers.
|
- Independent M1 review: **PASSED**; no concrete security or logic blockers.
|
||||||
- Reviewer suggestions: set `PAPER_COOKIE_SECURE=1` outside local HTTP; replace in-memory sessions before multi-worker deployment.
|
- Reviewer suggestions: set `PAPER_COOKIE_SECURE=1` outside local HTTP; replace in-memory sessions before multi-worker deployment.
|
||||||
- M1 reviewer backlog: add schema-drift, duplicate/reordered-row, and malformed-vintage regression fixtures.
|
- M1 reviewer backlog: add schema-drift, duplicate/reordered-row, and malformed-vintage regression fixtures.
|
||||||
@@ -51,4 +52,8 @@
|
|||||||
- Paper ledger persists atomically under ignored `backend/data/paper/ledger.json` when configured.
|
- Paper ledger persists atomically under ignored `backend/data/paper/ledger.json` when configured.
|
||||||
- Live API `0.5.0` created and replayed the same blocked Tourism research report; UI served the research-run panel and human-readable gate reason.
|
- Live API `0.5.0` created and replayed the same blocked Tourism research report; UI served the research-run panel and human-readable gate reason.
|
||||||
- Independent M2.5 review: **PASSED**; no concrete security or logic blockers.
|
- Independent M2.5 review: **PASSED**; no concrete security or logic blockers.
|
||||||
|
- Integrity hardening: normalized snapshots and raw payloads are bound to manifest metadata; malformed boolean flags, stale manifests, and cached-ready replay after tampering are rejected.
|
||||||
|
- Canonical hash algorithm is explicit: `sha256-json-canonical-v1` using sorted-key compact UTF-8 JSON after excluding only the normalized hash field.
|
||||||
|
- Integrity scope is local artifact/corruption detection. A hostile machine owner who can rewrite code, manifests, raw files and runtime environment is outside this local research app's threat model.
|
||||||
|
- Independent integrity-hardening review: **PASSED** under the stated local single-user threat model.
|
||||||
- Browser visual capture was blocked by Chrome remote-debugging permission; no permission dialog was clicked.
|
- Browser visual capture was blocked by Chrome remote-debugging permission; no permission dialog was clicked.
|
||||||
|
|||||||
74
docs/engineering-log/2026-08-23-integrity-hardening.md
Normal file
74
docs/engineering-log/2026-08-23-integrity-hardening.md
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
# 2026-08-23 — snapshot integrity hardening
|
||||||
|
|
||||||
|
## Plan status
|
||||||
|
|
||||||
|
- Root-cause fix for fail-open point-in-time metadata: complete.
|
||||||
|
- Raw payload and normalized snapshot binding: complete.
|
||||||
|
- Manifest/source metadata cross-checks: complete.
|
||||||
|
- Cached research report revalidation: complete.
|
||||||
|
- Trusted host/signing boundary: documented as local-only; multi-user deployment deferred.
|
||||||
|
|
||||||
|
## Changed files
|
||||||
|
|
||||||
|
- `backend/app/vintages.py` — canonical normalized hash, raw payload verification, manifest/source cross-checks, timezone-equivalent revision handling.
|
||||||
|
- `backend/app/prices.py` — canonical normalized hash, strict boolean point-in-time metadata, raw/manifest/source cross-checks.
|
||||||
|
- `backend/app/research.py` — validate selected snapshots before cached report lookup and derive the price gate from the verified snapshot source.
|
||||||
|
- `backend/app/__init__.py` — price health validates the latest snapshot before reporting availability.
|
||||||
|
- `backend/tests/test_vintages.py` — normalized payload tamper regression.
|
||||||
|
- `backend/tests/test_research.py` — manifest tamper and cached-ready replay regressions.
|
||||||
|
|
||||||
|
## Root cause
|
||||||
|
|
||||||
|
The previous implementation trusted manifest metadata too early. A truthy string such as `"false"` could pass a boolean gate, and a ready report could be returned from cache before current snapshot bytes were revalidated. The previous vintage loader also checked identity but not raw/normalized content against the manifest.
|
||||||
|
|
||||||
|
## Fix
|
||||||
|
|
||||||
|
All new persisted snapshots carry:
|
||||||
|
|
||||||
|
```text
|
||||||
|
normalized_hash_algorithm: sha256-json-canonical-v1
|
||||||
|
normalized_snapshot_hash: SHA-256(sorted-key compact UTF-8 JSON, excluding only the hash field)
|
||||||
|
raw_payload_hash: SHA-256(raw provider payload)
|
||||||
|
```
|
||||||
|
|
||||||
|
Loads now verify raw files, normalized content, manifest metadata, strict boolean types, and snapshot identity before the research runner can return a cached result or run an event study.
|
||||||
|
|
||||||
|
## Live evidence
|
||||||
|
|
||||||
|
```text
|
||||||
|
BOT source restarted and replayable=true.
|
||||||
|
Price snapshot re-collected under the new schema; 9 symbols; point_in_time=false.
|
||||||
|
GET /api/v1/prices/health → HTTP 200, available=true, revised_vendor_history.
|
||||||
|
POST /api/v1/research/tourism/run → HTTP 200, blocked 1/12 releases.
|
||||||
|
```
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
```text
|
||||||
|
PYTHONPATH=backend .venv/bin/python -W error -m unittest discover -s backend/tests -v
|
||||||
|
Ran 57 tests ... OK
|
||||||
|
|
||||||
|
npm run build
|
||||||
|
Vite build completed successfully.
|
||||||
|
|
||||||
|
npm audit --omit=dev --audit-level=high
|
||||||
|
found 0 vulnerabilities
|
||||||
|
|
||||||
|
python -m compileall -q backend
|
||||||
|
python /tmp/set50_platform_security_scan.py → {}
|
||||||
|
git diff --check
|
||||||
|
```
|
||||||
|
|
||||||
|
## Boundary
|
||||||
|
|
||||||
|
Content hashes are a local artifact/corruption control. They are not a signature against a hostile operator who can rewrite the application, manifests, raw files and runtime environment. Add a trusted deployment signing key before multi-user or hostile-host deployment.
|
||||||
|
|
||||||
|
## Independent review
|
||||||
|
|
||||||
|
```text
|
||||||
|
passed: true
|
||||||
|
security_concerns: []
|
||||||
|
logic_errors: []
|
||||||
|
```
|
||||||
|
|
||||||
|
Non-blocking backlog: retain the canonicalization/hash-version contract, keep crash/partial-write coverage, and introduce signed manifests when the deployment threat model expands.
|
||||||
48
docs/test-evidence/2026-08-23-integrity-hardening.md
Normal file
48
docs/test-evidence/2026-08-23-integrity-hardening.md
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
# Test evidence — 2026-08-23 integrity hardening
|
||||||
|
|
||||||
|
## Automated
|
||||||
|
|
||||||
|
```text
|
||||||
|
PYTHONPATH=backend .venv/bin/python -W error -m unittest discover -s backend/tests -v
|
||||||
|
Ran 57 tests ... OK
|
||||||
|
|
||||||
|
npm run build
|
||||||
|
Vite build completed successfully.
|
||||||
|
|
||||||
|
npm audit --omit=dev --audit-level=high
|
||||||
|
found 0 vulnerabilities
|
||||||
|
|
||||||
|
python -m compileall -q backend
|
||||||
|
python /tmp/set50_platform_security_scan.py
|
||||||
|
{}
|
||||||
|
|
||||||
|
git diff --check
|
||||||
|
```
|
||||||
|
|
||||||
|
## Regression coverage
|
||||||
|
|
||||||
|
- Tampered normalized Tourism snapshot is rejected.
|
||||||
|
- Tampered price manifest `point_in_time` metadata is rejected.
|
||||||
|
- A cached ready report is not replayed after price manifest tampering.
|
||||||
|
- Raw payload hash mismatch is rejected.
|
||||||
|
- Timezone-equivalent releases are treated as revisions.
|
||||||
|
- Weekend/holiday events use the first trading session after the event date.
|
||||||
|
|
||||||
|
## Live
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/v1/health → HTTP 200, version=0.5.0
|
||||||
|
GET /api/v1/prices/health → HTTP 200, available=true, point_in_time=false
|
||||||
|
GET /api/v1/data-health → HTTP 200, replayable=true
|
||||||
|
POST /api/v1/research/tourism/run → HTTP 200, blocked=1/12
|
||||||
|
```
|
||||||
|
|
||||||
|
## Independent review
|
||||||
|
|
||||||
|
```text
|
||||||
|
passed: true
|
||||||
|
security_concerns: []
|
||||||
|
logic_errors: []
|
||||||
|
```
|
||||||
|
|
||||||
|
Review passed under the stated local single-user threat model.
|
||||||
Reference in New Issue
Block a user