kunthawat
e04ea1dd27
- Full static mirror of the YellowPages/Drupal site (39 pages + all assets) - All URLs rewritten to relative/domain-agnostic form (works on any domain) - RFQ + request-quotation + contact forms converted to email (FormSubmit.co) - Google Maps embed replaced with static Google Maps iframe - nginx Dockerfile + config for EasyPanel deployment - mirror.py/rewrite.py/cleanup.py tooling for re-crawling updates
59 lines
1.9 KiB
Python
59 lines
1.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Post-process: re-rewrite rendered pages with the latest rewrite_rules (data-qrcodr etc.)
|
|
and download any newly-referenced assets. Idempotent."""
|
|
import os, re, queue, threading, urllib.parse, sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import mirror
|
|
from common import (BASE_URL, OUT_DIR, enqueue_asset, claim_asset_urls,
|
|
mark_asset_result, is_downloaded, assets_rel_for, safe_asset_path)
|
|
from rewrite import rewrite_html
|
|
|
|
def walk_pages():
|
|
"""Yield (page_url, abs_path) for every index.html under OUT_DIR."""
|
|
root = OUT_DIR
|
|
for dirpath, dirnames, filenames in os.walk(root):
|
|
if "index.html" in filenames:
|
|
rel = os.path.relpath(dirpath, root)
|
|
if rel == ".":
|
|
path = "/"
|
|
else:
|
|
path = "/" + rel.replace(os.sep, "/")
|
|
page_url = BASE_URL + path
|
|
yield page_url, os.path.join(dirpath, "index.html")
|
|
|
|
def main():
|
|
mirror.start_workers()
|
|
count = 0
|
|
changed = 0
|
|
for page_url, fpath in walk_pages():
|
|
count += 1
|
|
with open(fpath, "r", encoding="utf-8") as f:
|
|
text = f.read()
|
|
new = rewrite_html(text, page_url)
|
|
if new != text:
|
|
changed += 1
|
|
with open(fpath, "w", encoding="utf-8") as f:
|
|
f.write(new)
|
|
# drain any assets enqueued by this page's rewrite
|
|
batch = claim_asset_urls()
|
|
if batch:
|
|
mirror.submit_assets(batch)
|
|
# final drain
|
|
import time
|
|
while True:
|
|
batch = claim_asset_urls()
|
|
if batch:
|
|
mirror.submit_assets(batch)
|
|
if not mirror._submitted:
|
|
break
|
|
if all(is_downloaded(u) for u in list(mirror._submitted)):
|
|
if not claim_asset_urls():
|
|
break
|
|
time.sleep(0.3)
|
|
mirror.stop_workers()
|
|
print(f"Processed {count} pages, {changed} changed.")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|