123 lines
4.2 KiB
Python
123 lines
4.2 KiB
Python
"""File parsing for uploaded documents (pdf / markdown / txt)."""
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
from pathlib import Path
|
|
|
|
from ..config import Config
|
|
|
|
|
|
class ParseError(Exception):
|
|
pass
|
|
|
|
|
|
def _iter_bounded_page_text(page, fitz):
|
|
"""Extract a page in clipped bands so one page cannot materialize whole text."""
|
|
rect = getattr(page, "rect", None)
|
|
try:
|
|
x0 = float(getattr(rect, "x0"))
|
|
y0 = float(getattr(rect, "y0"))
|
|
x1 = float(getattr(rect, "x1"))
|
|
y1 = float(getattr(rect, "y1"))
|
|
except (AttributeError, TypeError, ValueError, OverflowError) as exc:
|
|
raise ParseError("PDF page geometry unavailable") from exc
|
|
height = y1 - y0
|
|
if (
|
|
not all(math.isfinite(value) for value in (x0, y0, x1, y1))
|
|
or x1 <= x0
|
|
or height <= 0
|
|
):
|
|
raise ParseError("PDF page geometry unavailable")
|
|
band_count = max(1, min(128, int(height / 32) + 1))
|
|
for index in range(band_count):
|
|
band_y0 = y0 + height * index / band_count
|
|
band_y1 = y0 + height * (index + 1) / band_count
|
|
clip = fitz.Rect(x0, band_y0, x1, band_y1)
|
|
chunk = page.get_text("text", clip=clip)
|
|
text = chunk if isinstance(chunk, str) else str(chunk)
|
|
max_chunk_chars = min(
|
|
Config.UPLOAD_MAX_EXTRACTED_CHARS,
|
|
Config.UPLOAD_MAX_PDF_CHUNK_CHARS,
|
|
)
|
|
if len(text) > max_chunk_chars:
|
|
raise ParseError("extracted text too large")
|
|
yield text
|
|
|
|
|
|
def parse_pdf(path: Path) -> str:
|
|
try:
|
|
import fitz # PyMuPDF
|
|
except ImportError as exc:
|
|
raise ParseError("PDF parser is unavailable") from exc
|
|
|
|
try:
|
|
too_large = path.exists() and path.stat().st_size > Config.UPLOAD_MAX_MB * 1024 * 1024
|
|
except OSError as exc:
|
|
raise ParseError("cannot inspect PDF") from exc
|
|
if too_large:
|
|
raise ParseError("PDF file too large")
|
|
try:
|
|
doc = fitz.open(path)
|
|
except Exception as exc:
|
|
raise ParseError("cannot open PDF") from exc
|
|
try:
|
|
page_count = getattr(doc, "page_count", None)
|
|
if page_count is None:
|
|
page_count = len(doc)
|
|
if int(page_count) > Config.UPLOAD_MAX_PDF_PAGES:
|
|
raise ParseError("PDF has too many pages")
|
|
parts = []
|
|
extracted_chars = 0
|
|
for page in doc:
|
|
for chunk in _iter_bounded_page_text(page, fitz):
|
|
text = chunk if isinstance(chunk, str) else str(chunk)
|
|
separator_chars = 1 if parts else 0
|
|
extracted_chars += separator_chars + len(text)
|
|
if extracted_chars > Config.UPLOAD_MAX_EXTRACTED_CHARS:
|
|
raise ParseError("extracted text too large")
|
|
parts.append(text)
|
|
return "\n".join(parts)
|
|
except ParseError:
|
|
raise
|
|
except Exception as exc:
|
|
raise ParseError("could not extract PDF text") from exc
|
|
finally:
|
|
doc.close()
|
|
|
|
|
|
def parse_text(path: Path) -> str:
|
|
try:
|
|
if path.stat().st_size > Config.UPLOAD_TEXT_MAX_BYTES:
|
|
raise ParseError("text document too large")
|
|
except FileNotFoundError as exc:
|
|
raise ParseError("text document is unavailable") from exc
|
|
raw = path.read_bytes()
|
|
if len(raw) > Config.UPLOAD_TEXT_MAX_BYTES:
|
|
raise ParseError("text document too large")
|
|
# Try utf-8 first, else detect encoding
|
|
try:
|
|
return raw.decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
pass
|
|
try:
|
|
from charset_normalizer import from_bytes
|
|
match = from_bytes(raw).best()
|
|
enc = getattr(match, "encoding", None) if match is not None else None
|
|
except ImportError as exc:
|
|
raise ParseError("text encoding detector is unavailable") from exc
|
|
if not enc:
|
|
raise ParseError("could not detect text encoding")
|
|
try:
|
|
return raw.decode(enc, errors="replace")
|
|
except (LookupError, UnicodeError) as exc:
|
|
raise ParseError("could not decode text document") from exc
|
|
|
|
|
|
def parse_document(path: Path) -> str:
|
|
ext = path.suffix.lower().lstrip(".")
|
|
if ext == "pdf":
|
|
return parse_pdf(path)
|
|
if ext not in {"md", "txt"}:
|
|
raise ParseError("unsupported document type")
|
|
return parse_text(path)
|