Mirror www.mitsuthailand.com as static site with dynamic URLs

- Full static mirror of the YellowPages/Drupal site (39 pages + all assets)
- All URLs rewritten to relative/domain-agnostic form (works on any domain)
- RFQ + request-quotation + contact forms converted to email (FormSubmit.co)
- Google Maps embed replaced with static Google Maps iframe
- nginx Dockerfile + config for EasyPanel deployment
- mirror.py/rewrite.py/cleanup.py tooling for re-crawling updates

postprocess.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""Post-process: re-rewrite rendered pages with the latest rewrite_rules (data-qrcodr etc.)
+and download any newly-referenced assets. Idempotent."""
+import os, re, queue, threading, urllib.parse, sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import mirror
+from common import (BASE_URL, OUT_DIR, enqueue_asset, claim_asset_urls,
+                    mark_asset_result, is_downloaded, assets_rel_for, safe_asset_path)
+from rewrite import rewrite_html
+
+def walk_pages():
+    """Yield (page_url, abs_path) for every index.html under OUT_DIR."""
+    root = OUT_DIR
+    for dirpath, dirnames, filenames in os.walk(root):
+        if "index.html" in filenames:
+            rel = os.path.relpath(dirpath, root)
+            if rel == ".":
+                path = "/"
+            else:
+                path = "/" + rel.replace(os.sep, "/")
+            page_url = BASE_URL + path
+            yield page_url, os.path.join(dirpath, "index.html")
+
+def main():
+    mirror.start_workers()
+    count = 0
+    changed = 0
+    for page_url, fpath in walk_pages():
+        count += 1
+        with open(fpath, "r", encoding="utf-8") as f:
+            text = f.read()
+        new = rewrite_html(text, page_url)
+        if new != text:
+            changed += 1
+            with open(fpath, "w", encoding="utf-8") as f:
+                f.write(new)
+        # drain any assets enqueued by this page's rewrite
+        batch = claim_asset_urls()
+        if batch:
+            mirror.submit_assets(batch)
+    # final drain
+    import time
+    while True:
+        batch = claim_asset_urls()
+        if batch:
+            mirror.submit_assets(batch)
+        if not mirror._submitted:
+            break
+        if all(is_downloaded(u) for u in list(mirror._submitted)):
+            if not claim_asset_urls():
+                break
+        time.sleep(0.3)
+    mirror.stop_workers()
+    print(f"Processed {count} pages, {changed} changed.")
+
+if __name__ == "__main__":
+    main()