#!/usr/bin/env python3 """Crawl www.mitsuthailand.com -> site/ as a static mirror with dynamic (relative) URLs.""" import os, re, queue, threading, urllib.parse, html import httpx from common import (BASE_HOST, BASE_URL, MEDIA_HOST, YP_HOST, OUT_DIR, assets_rel_for, safe_asset_path, enqueue_asset, claim_asset_urls, mark_asset_result, is_downloaded, canonical_page_path) client = httpx.Client( follow_redirects=True, timeout=40.0, headers={"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36", "Accept-Language": "th,en;q=0.9"}, ) # ---------------- URL classification ---------------- def resolve_url(raw, base): raw = html.unescape(raw).strip() if not raw or raw.startswith(("data:", "mailto:", "tel:", "javascript:", "#")): return None if raw.startswith(("//", "http")): parsed = urllib.parse.urlparse(raw) host = (parsed.hostname or "").lower() if host not in (BASE_HOST, MEDIA_HOST, YP_HOST): return None if "/lp/" in parsed.path or parsed.path.startswith("/lp"): return None return urllib.parse.urlunparse(parsed._replace(scheme="https")) if parsed.scheme in ("", "http") else raw absurl = urllib.parse.urljoin(base, raw) parsed = urllib.parse.urlparse(absurl) host = (parsed.hostname or "").lower() if host and host not in (BASE_HOST, MEDIA_HOST, YP_HOST): return None if "/lp/" in parsed.path or parsed.path.startswith("/lp"): return None return absurl def is_page_url(url): parsed = urllib.parse.urlparse(url) host = (parsed.hostname or "").lower() # The www.yellowpages.co.th/lp/... pages are a YellowPages mirror of the same content # and will die too; skip them. Only mirror the main mitsuthailand.com pages. if host == YP_HOST: return False path = parsed.path.rstrip("/") # Skip /lp/... "live preview" duplicate copies of the same content (very long paths). if "/lp/" in path or path.startswith("/lp"): return False # skip Drupal file assets (they have extensions or are /sites /core /themes /system paths) if path.startswith(("/sites/", "/core/", "/themes/", "/system/", "/sites")): return False ext = os.path.splitext(path)[1].lower() # Drupal content routes (no file extension) are pages return ext in (".html", "") # ---------------- asset workers ---------------- _jobs = queue.Queue() _shutdown = threading.Event() _submitted = set() # URLs already pushed to _jobs (guarantees no dup put) _sub_lock = threading.Lock() def start_workers(n=12): for _ in range(n): t = threading.Thread(target=_worker, daemon=True) t.start() def stop_workers(): _shutdown.set() def submit_assets(urls): """Push each URL to _jobs exactly once; returns count of new submissions.""" new = [] with _sub_lock: for u in urls: if u not in _submitted and not is_downloaded(u): _submitted.add(u) new.append(u) for u in new: _jobs.put(u) return len(new) def _worker(): while not _shutdown.is_set(): try: url = _jobs.get(timeout=0.5) except queue.Empty: continue try: r = client.get(url) if r.status_code != 200: print(f" [asset {r.status_code}] {url}") mark_asset_result(url, None) continue ctype = r.headers.get("content-type", "") data = r.content rel = safe_asset_path(assets_rel_for(url), url) dest = os.path.join(OUT_DIR, rel) os.makedirs(os.path.dirname(dest), exist_ok=True) if "css" in ctype or url.rstrip("?;").endswith(".css"): try: text = data.decode("utf-8", "replace") text = rewrite_css(url, text) data = text.encode("utf-8") except Exception as e: print(" [css err]", e) with open(dest, "wb") as f: f.write(data) mark_asset_result(url, rel) except Exception as e: print(f" [asset ERR] {url}: {e}") mark_asset_result(url, None) # ---------------- CSS rewriting ---------------- def rewrite_css(css_url, css_text): def repl(m): u = m.group(1).strip() if u.startswith(("data:", "#", "http", "//")) or u.startswith("url("): return m.group(0) abs = urllib.parse.urljoin(css_url, u) parsed = urllib.parse.urlparse(abs) host = (parsed.hostname or "").lower() if host and host not in (BASE_HOST, MEDIA_HOST, YP_HOST): return m.group(0) rel = safe_asset_path(assets_rel_for(abs), abs) enqueue_asset(abs) # css files live under assets//..., reference is relative to that file's dir css_dir = os.path.dirname(assets_rel_for(css_url)) # e.g. assets/sites/storage/files/css from_dir = os.path.join(OUT_DIR, css_dir) rel_from = os.path.relpath(os.path.join(OUT_DIR, rel), from_dir) return 'url("{}")'.format(rel_from) return re.sub(r'url\(\s*["\']?([^"\'()]+)["\']?\s*\)', repl, css_text) # ---------------- crawl + render ---------------- def fetch_sitemap_urls(): urls = {BASE_URL, BASE_URL + "/"} try: r = client.get(f"{BASE_URL}/sitemap.xml") if r.status_code == 200: for m in re.finditer(r"([^<]+)", r.text): u = m.group(1).strip().replace("http://", "https://") if BASE_HOST in u: urls.add(u) except Exception as e: print("sitemap err", e) return urls def crawl_pages(seed): crawled = {} # canonical path -> representative URL to_crawl = queue.Queue() for u in seed: to_crawl.put(u) for _ in range(5): found = {} while not to_crawl.empty(): try: url = to_crawl.get_nowait() except queue.Empty: break c_key = canonical_page_path(url) if c_key in crawled: continue crawled[c_key] = url try: r = client.get(url) except Exception as e: print(" [page ERR]", url, e) continue if r.status_code != 200: print(" [page", r.status_code, "]", url) continue text = r.text for l in re.findall(r'(?:href|src|action)="([^"]+)"', text): a = resolve_url(l, url) if a is None: continue if is_page_url(a): ca = canonical_page_path(a) if ca not in crawled and ca not in found: found[ca] = a else: enqueue_asset(a) for ck, u in found.items(): to_crawl.put(u) print(f" pass: {len(crawled)} pages, +{len(found)} new") if not found: break return list(crawled.values()) def render_pages(pages): from rewrite import rewrite_html seen = set() for url in sorted(pages): try: canon = canonical_page_path(url) if canon in seen: continue seen.add(canon) r = client.get(url) if r.status_code != 200: print(" [render", r.status_code, "]", url) continue text = rewrite_html(r.text, url) rel_path = canon.lstrip("/") dest = os.path.join(OUT_DIR, rel_path, "index.html") os.makedirs(os.path.dirname(dest), exist_ok=True) with open(dest, "w", encoding="utf-8") as f: f.write(text) print(f" [page OK] {canon}") except Exception as e: print(" [render ERR]", url, e) def drain_assets(): """Pull newly-queued assets (common._queue) into _jobs until nothing is pending or in-flight. Returns when all submitted assets have a result AND no new items were enqueued in the meantime.""" import time as _time idle_rounds = 0 while True: batch = claim_asset_urls() if batch: submit_assets(batch) idle_rounds = 0 if not _submitted: return if all(is_downloaded(u) for u in list(_submitted)): # confirm nothing new got queued before returning if not claim_asset_urls(): return _time.sleep(0.3) def main(): print("Mirroring", BASE_URL, "->", OUT_DIR) seed = fetch_sitemap_urls() print(f"Seeded {len(seed)} URLs.") pages = crawl_pages(seed) print(f"Total pages: {len(pages)}") start_workers() drain_assets() render_pages(pages) drain_assets() stop_workers() print(f"Pages rendered: {len(pages)}") print("Done. Site in", OUT_DIR) if __name__ == "__main__": main()