Files
thailand-water/scripts/import_dpm_boundaries.py
T

95 lines
4.2 KiB
Python

"""Import official DPM province, amphoe and tambon polygons into PostGIS."""
from __future__ import annotations
import json
import argparse
from urllib.parse import urlencode
from urllib.request import Request, urlopen
import psycopg
from app.settings import configured_database_url
BASE = "https://gis-portal.disaster.go.th/arcgis/rest/services/MapDX/DPM_TH_Boundary/FeatureServer"
LAYERS = (
(1, "admin_province", ("PROV_CODE", "PROV_NAM_T")),
(2, "admin_amphoe", ("AMP_CODE", "PROV_CODE", "PROV_NAM_T", "AMP_NAM_T")),
(3, "admin_tambon", ("TAM_CODE", "AMP_CODE", "PROV_CODE", "PROV_NAM_T", "AMP_NAM_T", "TAM_NAM_T")),
)
FIELDS = {
"admin_province": ("prov_code", "prov_name_th"),
"admin_amphoe": ("amp_code", "prov_code", "prov_name_th", "amp_name_th"),
"admin_tambon": ("tam_code", "amp_code", "prov_code", "prov_name_th", "amp_name_th", "tam_name_th"),
}
def _request(layer: int, **params) -> dict:
url = f"{BASE}/{layer}/query?" + urlencode({**params, "f": params.pop("f", "json")})
with urlopen(Request(url, headers={"User-Agent": "ThailandFloodIntelligence/0.1"}), timeout=45) as response:
payload = json.load(response)
if "error" in payload:
raise RuntimeError(f"DPM layer {layer}: {payload['error'].get('message')}")
return payload
def _import_layer(connection: psycopg.Connection, layer: int, table: str, source_fields: tuple[str, ...]) -> int:
ids = _request(layer, where="1=1", returnIdsOnly="true").get("objectIds") or []
if not ids:
raise RuntimeError(f"DPM layer {layer} returned no IDs")
columns = FIELDS[table]
placeholders = ",".join(["%s"] * len(columns))
names = ",".join(columns)
primary = columns[0]
updates = ",".join(f"{name}=EXCLUDED.{name}" for name in columns[1:])
merged_geom = (f"ST_Multi(ST_CollectionExtract(ST_UnaryUnion(ST_Collect({table}.geom,"
"EXCLUDED.geom)),3))" if table == "admin_tambon" else "EXCLUDED.geom")
sql = (f"INSERT INTO {table} ({names},geom,source_url) VALUES ({placeholders},"
"ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_SetSRID(ST_GeomFromGeoJSON(%s),4326)),3)),%s) "
f"ON CONFLICT ({primary}) DO UPDATE SET {updates},geom={merged_geom},"
"source_url=EXCLUDED.source_url,imported_at=now()")
imported = 0
seen: dict[str, tuple[str, ...]] = {}
for offset in range(0, len(ids), 75):
chunk = ids[offset:offset + 75]
data = _request(layer, objectIds=",".join(map(str, chunk)), outFields=",".join(source_fields),
outSR="4326", f="geojson")
features = data.get("features") or []
if len(features) != len(chunk):
raise RuntimeError(f"DPM layer {layer}: expected {len(chunk)} shapes, got {len(features)}")
rows = []
for feature in features:
properties = feature.get("properties") or {}
values = [str(properties.get(field) or "").strip() for field in source_fields]
if not all(values) or not feature.get("geometry"):
raise RuntimeError(f"DPM layer {layer}: missing required code/name/geometry")
if values[0] in seen and seen[values[0]] != tuple(values[1:]):
raise RuntimeError(f"DPM layer {layer}: code {values[0]} maps to different names")
seen[values[0]] = tuple(values[1:])
rows.append((*values, json.dumps(feature["geometry"], ensure_ascii=False), f"{BASE}/{layer}"))
with connection.transaction():
with connection.cursor() as cursor:
cursor.executemany(sql, rows)
imported += len(rows)
print(f"{table}: {imported}/{len(ids)}", flush=True)
return imported
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--layer", type=int, choices=(1, 2, 3))
args = parser.parse_args()
url = configured_database_url()
if not url:
raise RuntimeError("DATABASE_URL is required")
with psycopg.connect(url, autocommit=True) as connection:
for layer, table, source_fields in LAYERS:
if args.layer and layer != args.layer:
continue
count = _import_layer(connection, layer, table, source_fields)
print(f"Imported {table}: {count}", flush=True)
if __name__ == "__main__":
main()