kunthawat
e04ea1dd27
- Full static mirror of the YellowPages/Drupal site (39 pages + all assets) - All URLs rewritten to relative/domain-agnostic form (works on any domain) - RFQ + request-quotation + contact forms converted to email (FormSubmit.co) - Google Maps embed replaced with static Google Maps iframe - nginx Dockerfile + config for EasyPanel deployment - mirror.py/rewrite.py/cleanup.py tooling for re-crawling updates
112 lines
4.5 KiB
Python
112 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""HTML rewriting: domain-agnostic relative URLs + RFQ form -> email (FormSubmit)."""
|
|
import os, re, urllib.parse, html as htmllib
|
|
|
|
from common import (BASE_HOST, MEDIA_HOST, YP_HOST, OUT_DIR,
|
|
assets_rel_for, safe_asset_path, enqueue_asset,
|
|
canonical_page_path)
|
|
|
|
FORM_RECIPIENT = "mail@mitsuchaiyaporn.com"
|
|
|
|
|
|
def _rel_for_page(url):
|
|
p = urllib.parse.urlparse(url)
|
|
path = p.path.rstrip("/")
|
|
return "/" if path == "" else path
|
|
|
|
def _depth(url):
|
|
p = urllib.parse.urlparse(url)
|
|
path = p.path.rstrip("/")
|
|
return 0 if path == "" else path.count("/")
|
|
|
|
def _root_prefix(url):
|
|
return "../" * _depth(url)
|
|
|
|
def _abs(u, page_url):
|
|
if u.startswith("//"):
|
|
return "https:" + u
|
|
if u.startswith("http"):
|
|
return u
|
|
base = "https://" + BASE_HOST + (_rel_for_page(page_url).rstrip("/") + "/" if _rel_for_page(page_url) != "/" else "/")
|
|
return urllib.parse.urljoin(base, u)
|
|
|
|
|
|
def rewrite_url_value(raw, page_url):
|
|
raw = htmllib.unescape(raw).strip()
|
|
if not raw or raw.startswith(("data:", "mailto:", "tel:", "javascript:", "#")):
|
|
return raw
|
|
parsed = urllib.parse.urlparse(_abs(raw, page_url))
|
|
host = (parsed.hostname or "").lower()
|
|
if host == BASE_HOST:
|
|
# decode path to the canonical (decoded) form so links match stored dirs
|
|
canon = urllib.parse.unquote(parsed.path).rstrip("/")
|
|
prefix = _root_prefix(page_url)
|
|
suffix = ("?" + parsed.query) if parsed.query else ""
|
|
suffix += ("#" + parsed.fragment) if parsed.fragment else ""
|
|
if canon == "":
|
|
return prefix + suffix
|
|
return prefix + canon.lstrip("/") + suffix
|
|
if host in (MEDIA_HOST, YP_HOST):
|
|
absurl = urllib.parse.urlunparse(parsed)
|
|
rel = safe_asset_path(assets_rel_for(absurl), absurl)
|
|
enqueue_asset(absurl)
|
|
return _root_prefix(page_url) + rel
|
|
return raw
|
|
|
|
|
|
def rewrite_css_asset_links(text, page_url):
|
|
def repl(m):
|
|
u = m.group(1).strip()
|
|
if u.startswith(("data:", "#", "http", "//")):
|
|
return m.group(0)
|
|
abs = _abs(u, page_url)
|
|
parsed = urllib.parse.urlparse(abs)
|
|
host = (parsed.hostname or "").lower()
|
|
if host not in (BASE_HOST, MEDIA_HOST, YP_HOST):
|
|
return m.group(0)
|
|
rel = safe_asset_path(assets_rel_for(abs), abs)
|
|
enqueue_asset(abs)
|
|
return 'url("{}")'.format(_root_prefix(page_url) + rel)
|
|
return re.sub(r'url\(\s*["\']?([^"\'()]+)["\']?\s*\)', repl, text)
|
|
|
|
|
|
def _action_is_backend(action):
|
|
a = action.lower()
|
|
return ("/rfq" in a) or ("magic-contact" in a) or ("/contactus" in a)
|
|
|
|
|
|
def convert_rfq_form(html, page_url):
|
|
pattern = re.compile(r'(<form\b[^>]*>)(.*?)(</form>)', re.S)
|
|
def repl(m):
|
|
open_tag, inner, close_tag = m.group(1), m.group(2), m.group(3)
|
|
action_m = re.search(r'action="([^"]*)"', open_tag)
|
|
if not action_m:
|
|
return m.group(0)
|
|
if not _action_is_backend(_abs(action_m.group(1), page_url)):
|
|
return m.group(0)
|
|
fs_action = "https://formsubmit.co/ajax/" + FORM_RECIPIENT
|
|
new_open = '<form class="typ-static-form" action="' + fs_action + '" method="POST">'
|
|
hidden = ('<input type="hidden" name="_subject" value="แบบฟอร์มติดต่อจากเว็บ mitsuchaiyaporn">'
|
|
'<input type="hidden" name="_template" value="table">'
|
|
'<input type="hidden" name="_captcha" value="false">')
|
|
return new_open + hidden + inner + close_tag
|
|
return pattern.sub(repl, html)
|
|
|
|
|
|
def rewrite_html(text, page_url):
|
|
text = re.sub(r'<link rel="canonical"[^>]*>', '', text)
|
|
text = re.sub(r'<meta property="og:url"[^>]*>', '', text)
|
|
text = re.sub(r'<meta name="twitter:url"[^>]*>', '', text)
|
|
text = convert_rfq_form(text, page_url)
|
|
text = re.sub(r'href="([^"]*)"',
|
|
lambda m: 'href="' + rewrite_url_value(m.group(1), page_url) + '"', text)
|
|
text = re.sub(r'src="([^"]*)"',
|
|
lambda m: 'src="' + rewrite_url_value(m.group(1), page_url) + '"', text)
|
|
# QR-code / other data-* URL attributes pointing at media host -> localize
|
|
text = re.sub(r'(data-qrcodr|data-\w*url)="([^"]*)"',
|
|
lambda m: m.group(1) + '="' + rewrite_url_value(m.group(2), page_url) + '"', text)
|
|
text = re.sub(r'action="([^"]*)"',
|
|
lambda m: 'action="' + rewrite_url_value(m.group(1), page_url) + '"', text)
|
|
text = rewrite_css_asset_links(text, page_url)
|
|
return text
|