kunthawat
e04ea1dd27
- Full static mirror of the YellowPages/Drupal site (39 pages + all assets) - All URLs rewritten to relative/domain-agnostic form (works on any domain) - RFQ + request-quotation + contact forms converted to email (FormSubmit.co) - Google Maps embed replaced with static Google Maps iframe - nginx Dockerfile + config for EasyPanel deployment - mirror.py/rewrite.py/cleanup.py tooling for re-crawling updates
87 lines
3.3 KiB
Python
87 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Repair pass: fix HTML references that point at files which exist under a
|
|
different (shorter/aliased) name on disk. Drupal style-derived images are
|
|
referenced by long decorative names (e.g. 68997caf12e42-Mitsubishi+Attrage+
|
|
....jpg) but the actual downloaded file uses the base id (68997caf12e42.jpg).
|
|
|
|
For each unresolvable local reference, try:
|
|
1. Match by (dir, base-id-before-minus/plus, ext) -> existing file
|
|
2. Match by (dir, full stem, ext)
|
|
If found, rewrite the reference to the existing file.
|
|
"""
|
|
import os, re, sys, urllib.parse
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
from common import OUT_DIR
|
|
|
|
|
|
def build_file_index():
|
|
index = {}
|
|
for dirpath, dirnames, filenames in os.walk(OUT_DIR):
|
|
for fn in filenames:
|
|
if fn == "index.html":
|
|
continue
|
|
rel = os.path.relpath(os.path.join(dirpath, fn), OUT_DIR)
|
|
stem = os.path.splitext(fn)[0]
|
|
ext = os.path.splitext(fn)[1].lower()
|
|
d = os.path.normpath(dirpath)
|
|
index[(d, stem.lower(), ext)] = rel
|
|
baseid = re.split(r'[-+]', stem)[0]
|
|
if len(baseid) >= 6:
|
|
index.setdefault((d, baseid, ext), rel)
|
|
return index
|
|
|
|
|
|
def repair():
|
|
index = build_file_index()
|
|
print(f"Indexed {len(index)} asset aliases.")
|
|
all_missing = 0
|
|
fixed = 0
|
|
for dirpath, dirnames, filenames in os.walk(OUT_DIR):
|
|
if "index.html" not in filenames:
|
|
continue
|
|
fpath = os.path.join(dirpath, "index.html")
|
|
with open(fpath, encoding="utf-8") as f:
|
|
html = f.read()
|
|
changed = False
|
|
|
|
def fix_ref(m):
|
|
nonlocal all_missing, fixed, changed
|
|
prefix, u = m.group(1), m.group(2)
|
|
upath = u.split("?")[0].split("#")[0]
|
|
if not upath or upath.startswith(("http", "data:", "mailto:", "tel:",
|
|
"javascript:", "//", "#")):
|
|
return m.group(0)
|
|
target = os.path.normpath(os.path.join(dirpath, upath))
|
|
if os.path.exists(target):
|
|
return m.group(0)
|
|
all_missing += 1
|
|
tdir = os.path.dirname(target)
|
|
tstem = os.path.splitext(os.path.basename(target))[0]
|
|
text = os.path.splitext(os.path.basename(target))[1].lower()
|
|
baseid = re.split(r'[-+]', tstem)[0]
|
|
cand = None
|
|
if baseid and len(baseid) >= 6:
|
|
cand = index.get((os.path.normpath(tdir), baseid, text))
|
|
if cand is None:
|
|
cand = index.get((os.path.normpath(tdir), tstem.lower(), text))
|
|
if cand:
|
|
newu = os.path.relpath(os.path.join(OUT_DIR, cand), dirpath).replace(os.sep, "/")
|
|
fixed += 1
|
|
changed = True
|
|
q = "?" + u.split("?")[1] if "?" in u else ""
|
|
h = "#" + u.split("#")[1] if "#" in u else ""
|
|
return prefix + newu + q + h
|
|
return m.group(0)
|
|
|
|
html = re.sub(r'(src|href)="([^"]*)"', fix_ref, html)
|
|
if changed:
|
|
with open(fpath, "w", encoding="utf-8") as f:
|
|
f.write(html)
|
|
print(f"Total unresolvable local refs: {all_missing}; fixed {fixed}.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
repair()
|