Files
thailand-water/scripts/import_hydro_vectors.py
T

216 lines
12 KiB
Python

"""Import full Asia HydroBASINS/HydroRIVERS topology into PostGIS."""
from __future__ import annotations
import hashlib
import json
import re
import uuid
from pathlib import Path
from typing import Any, Iterator
from app.settings import configured_database_url
def _drivers() -> tuple[Any, Any]:
try:
from osgeo import ogr
from psycopg.types.json import Jsonb
except ImportError as exc:
raise RuntimeError("Install GDAL/OGR Python bindings and psycopg before importing vector products") from exc
return ogr, Jsonb
def _connection():
try:
import psycopg
except ImportError as exc:
raise RuntimeError("Install psycopg before importing vector products") from exc
url = configured_database_url()
if not url:
raise RuntimeError("Set DATABASE_URL or DATABASE_DSN before importing datasets")
return psycopg.connect(url)
def _properties(feature: Any) -> dict[str, Any]:
definition = feature.GetDefnRef()
values = {}
for index in range(definition.GetFieldCount()):
name = definition.GetFieldDefn(index).GetName()
value = feature.GetField(index)
values[name] = value
return values
def _field(properties: dict[str, Any], *names: str) -> Any:
values = {str(key).upper(): value for key, value in properties.items()}
for name in names:
if name.upper() in values:
return values[name.upper()]
return None
def _json_geometry(feature: Any) -> str:
geometry = feature.GetGeometryRef()
if geometry is None:
raise ValueError("vector feature is missing its geometry")
return geometry.ExportToJson()
def _source_epsg(layer: Any) -> int:
spatial_ref = layer.GetSpatialRef()
epsg = int(spatial_ref.GetAuthorityCode(None)) if spatial_ref and spatial_ref.GetAuthorityCode(None) else 0
if epsg not in {4326, 4269}:
raise ValueError(f"unexpected vector CRS EPSG:{epsg}")
return epsg
def _insert_schema_version(cursor: Any, source_id: str, files: list[Path], properties: dict[str, Any]) -> None:
hasher = hashlib.sha256()
for path in sorted(files):
hasher.update(path.name.encode())
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
hasher.update(chunk)
version = f"sha256:{hasher.hexdigest()}"
from psycopg.types.json import Jsonb
cursor.execute(
"""INSERT INTO source_schema_versions(source_id,schema_version,valid_from,schema_definition,parser_version)
VALUES (%s,%s,now(),%s,'1') ON CONFLICT(source_id,schema_version) DO NOTHING""",
(source_id, version, Jsonb({**properties, "source_files": [path.name for path in files]})),
)
def import_hydrobasins(report: dict[str, Any]) -> dict[str, Any]:
ogr, Jsonb = _drivers()
vector_files = [Path(item["path"]) for item in report.get("vector_files", [])]
if not report.get("valid") or not vector_files:
raise ValueError("HydroBASINS files did not pass CRS, geometry and topology validation")
counts: dict[int, int] = {}
with _connection() as connection:
with connection.cursor() as cursor:
for path in vector_files:
dataset = ogr.Open(str(path), 0)
if dataset is None:
raise ValueError(f"Cannot open {path.name}")
layer = dataset.GetLayer(0)
epsg = _source_epsg(layer)
filename_level = re.search(r"(?:lev|level)[_-]?(\d{1,2})", path.stem, re.I)
for feature in layer:
props = _properties(feature)
basin_id = _field(props, "HYBAS_ID")
if basin_id is None:
raise ValueError(f"{path.name}: missing HYBAS_ID")
level = _field(props, "LEVEL")
level = int(level) if level is not None else int(filename_level.group(1)) if filename_level else None
if level not in range(1, 13):
raise ValueError(f"{path.name}: HydroBASINS level could not be verified as 1-12")
name_en = _field(props, "NAME_EN", "NAME")
cursor.execute(
"""INSERT INTO basins(basin_id,name_en,source,source_id,level,geom,properties)
VALUES (%s,%s,'HydroBASINS','hydrobasins',%s,
ST_Multi(ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(%s),%s),4326)),%s)
ON CONFLICT (basin_id) DO UPDATE SET name_en=EXCLUDED.name_en,
source=EXCLUDED.source,source_id=EXCLUDED.source_id,level=EXCLUDED.level,
geom=EXCLUDED.geom,properties=EXCLUDED.properties""",
(str(basin_id), str(name_en) if name_en is not None else None, level, _json_geometry(feature), epsg, Jsonb(props)),
)
if level in {8, 9, 10}:
area = _field(props, "SUB_AREA", "AREA_SQKM", "CATCH_SKM")
cursor.execute(
"""INSERT INTO catchments
(catchment_id,basin_id,source,source_id,area_km2,transboundary,geom,properties)
VALUES (%s,%s,'HydroBASINS','hydrobasins',%s,false,
ST_Multi(ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(%s),%s),4326)),%s)
ON CONFLICT (catchment_id) DO UPDATE SET basin_id=EXCLUDED.basin_id,
area_km2=EXCLUDED.area_km2,geom=EXCLUDED.geom,properties=EXCLUDED.properties""",
(str(basin_id), str(basin_id), float(area) if area is not None else None,
_json_geometry(feature), epsg,
Jsonb({**props, "transboundary_status": "NOT_EVALUATED; preserve Asia geometry and upstream topology"})),
)
counts[level] = counts.get(level, 0) + 1
missing = set(range(1, 13)) - set(counts)
if missing:
raise ValueError(f"Full Asia HydroBASINS import requires levels 1-12; missing {sorted(missing)}")
_insert_schema_version(cursor, "hydrobasins", vector_files, {"levels": counts, "extent_import": "full source files; no Thailand-border clipping"})
return {"source_id": "hydrobasins", "basin_levels": counts, "rows": sum(counts.values()), "extent_import": "full source files"}
def import_hydrorivers(report: dict[str, Any]) -> dict[str, Any]:
ogr, Jsonb = _drivers()
vector_files = [Path(item["path"]) for item in report.get("vector_files", [])]
if not report.get("valid") or not vector_files:
raise ValueError("HydroRIVERS files did not pass CRS, geometry and topology validation")
rows: list[dict[str, Any]] = []
for path in vector_files:
dataset = ogr.Open(str(path), 0)
if dataset is None:
raise ValueError(f"Cannot open {path.name}")
layer = dataset.GetLayer(0)
epsg = _source_epsg(layer)
for feature in layer:
props = _properties(feature)
reach_id = _field(props, "HYRIV_ID")
next_down = _field(props, "NEXT_DOWN")
if reach_id is None or next_down is None:
raise ValueError(f"{path.name}: HydroRIVERS topology fields are missing")
geometry = json.loads(_json_geometry(feature))
coords = _line_ends(geometry)
rows.append({"id": str(reach_id), "next": str(next_down), "props": props, "geometry": geometry, "ends": coords, "epsg": epsg})
if not rows:
raise ValueError("HydroRIVERS Asia file contained no reaches")
known = {row["id"] for row in rows}
absent_downstream = sorted({row["next"] for row in rows if row["next"] not in {"0", "", "None"} and row["next"] not in known})
if absent_downstream:
raise ValueError(f"HydroRIVERS topology is incomplete: {len(absent_downstream)} NEXT_DOWN targets are absent from the imported Asia product")
node_ids = {row["id"]: uuid.uuid5(uuid.NAMESPACE_URL, f"hydrorivers:reach:{row['id']}") for row in rows}
outlet_ids = {row["id"]: uuid.uuid5(uuid.NAMESPACE_URL, f"hydrorivers:outlet:{row['id']}") for row in rows if row["next"] in {"0", "", "None"}}
with _connection() as connection:
with connection.cursor() as cursor:
for row in rows:
lon, lat = row["ends"][0]
cursor.execute(
"""INSERT INTO network_nodes(node_id,node_type,canonical_name,provider,provider_id,geom,confidence,properties)
VALUES (%s,'JUNCTION',NULL,'HydroRIVERS',%s,
ST_Transform(ST_SetSRID(ST_MakePoint(%s,%s),%s),4326),0.35,%s)
ON CONFLICT (node_id) DO UPDATE SET geom=EXCLUDED.geom,properties=EXCLUDED.properties""",
(node_ids[row["id"]], row["id"], lon, lat, row["epsg"], Jsonb({"source_fields": row["props"], "topology_status": "source-derived; hydraulic direction remains dynamic"})),
)
if row["id"] in outlet_ids:
end_lon, end_lat = row["ends"][1]
cursor.execute(
"""INSERT INTO network_nodes(node_id,node_type,provider,provider_id,geom,confidence,properties)
VALUES (%s,'OUTLET','HydroRIVERS',%s,
ST_Transform(ST_SetSRID(ST_MakePoint(%s,%s),%s),4326),0.35,%s)
ON CONFLICT (node_id) DO UPDATE SET geom=EXCLUDED.geom,properties=EXCLUDED.properties""",
(outlet_ids[row["id"]], f"outlet:{row['id']}", end_lon, end_lat, row["epsg"], Jsonb({"source_reach_id": row["id"], "topology_status": "natural-river terminal in source product"})),
)
for row in rows:
target_id = outlet_ids[row["id"]] if row["id"] in outlet_ids else node_ids[row["next"]]
reach = str(_field(row["props"], "HYRIV_ID"))
length = _field(row["props"], "LENGTH_KM")
edge_id = uuid.uuid5(uuid.NAMESPACE_URL, f"hydrorivers:edge:{reach}")
cursor.execute(
"""INSERT INTO network_edges(edge_id,edge_type,canonical_name,from_node_id,to_node_id,direction_type,
geom,length_m,network_confidence,hydraulic_parameters_verified,properties)
VALUES (%s,'RIVER_REACH',NULL,%s,%s,'DYNAMIC',
ST_Multi(ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(%s),%s),4326)),%s,0.35,false,%s)
ON CONFLICT (edge_id) DO UPDATE SET from_node_id=EXCLUDED.from_node_id,to_node_id=EXCLUDED.to_node_id,
direction_type='DYNAMIC',geom=EXCLUDED.geom,length_m=EXCLUDED.length_m,
network_confidence=EXCLUDED.network_confidence,hydraulic_parameters_verified=false,properties=EXCLUDED.properties""",
(edge_id, node_ids[reach], target_id, json.dumps(row["geometry"]), row["epsg"], float(length) * 1000 if length is not None else None,
Jsonb({"source": "HydroRIVERS", "source_id": "hydrorivers", "source_fields": row["props"], "next_down_provider_id": row["next"], "hydraulic_parameters_verified": False, "artificial_canal": False})),
)
_insert_schema_version(cursor, "hydrorivers", vector_files, {"reach_count": len(rows), "topology_field": "NEXT_DOWN", "artificial_canal_inference": False})
return {"source_id": "hydrorivers", "reaches": len(rows), "nodes": len(node_ids) + len(outlet_ids), "network_confidence": 0.35, "direction_type": "DYNAMIC", "hydraulic_parameters_verified": False}
def _line_ends(geometry: dict[str, Any]) -> tuple[tuple[float, float], tuple[float, float]]:
coordinates = geometry.get("coordinates")
if geometry.get("type") == "LineString" and coordinates and len(coordinates) >= 2:
return (tuple(coordinates[0][:2]), tuple(coordinates[-1][:2]))
if geometry.get("type") == "MultiLineString" and coordinates and coordinates[0] and len(coordinates[0]) >= 2:
return (tuple(coordinates[0][0][:2]), tuple(coordinates[-1][-1][:2]))
raise ValueError("HydroRIVERS reach geometry is not a valid line")