Files
sales-trainer/backend/app/services/file_parser.py
T

123 lines
4.2 KiB
Python

"""File parsing for uploaded documents (pdf / markdown / txt)."""
from __future__ import annotations
import math
from pathlib import Path
from ..config import Config
class ParseError(Exception):
pass
def _iter_bounded_page_text(page, fitz):
"""Extract a page in clipped bands so one page cannot materialize whole text."""
rect = getattr(page, "rect", None)
try:
x0 = float(getattr(rect, "x0"))
y0 = float(getattr(rect, "y0"))
x1 = float(getattr(rect, "x1"))
y1 = float(getattr(rect, "y1"))
except (AttributeError, TypeError, ValueError, OverflowError) as exc:
raise ParseError("PDF page geometry unavailable") from exc
height = y1 - y0
if (
not all(math.isfinite(value) for value in (x0, y0, x1, y1))
or x1 <= x0
or height <= 0
):
raise ParseError("PDF page geometry unavailable")
band_count = max(1, min(128, int(height / 32) + 1))
for index in range(band_count):
band_y0 = y0 + height * index / band_count
band_y1 = y0 + height * (index + 1) / band_count
clip = fitz.Rect(x0, band_y0, x1, band_y1)
chunk = page.get_text("text", clip=clip)
text = chunk if isinstance(chunk, str) else str(chunk)
max_chunk_chars = min(
Config.UPLOAD_MAX_EXTRACTED_CHARS,
Config.UPLOAD_MAX_PDF_CHUNK_CHARS,
)
if len(text) > max_chunk_chars:
raise ParseError("extracted text too large")
yield text
def parse_pdf(path: Path) -> str:
try:
import fitz # PyMuPDF
except ImportError as exc:
raise ParseError("PDF parser is unavailable") from exc
try:
too_large = path.exists() and path.stat().st_size > Config.UPLOAD_MAX_MB * 1024 * 1024
except OSError as exc:
raise ParseError("cannot inspect PDF") from exc
if too_large:
raise ParseError("PDF file too large")
try:
doc = fitz.open(path)
except Exception as exc:
raise ParseError("cannot open PDF") from exc
try:
page_count = getattr(doc, "page_count", None)
if page_count is None:
page_count = len(doc)
if int(page_count) > Config.UPLOAD_MAX_PDF_PAGES:
raise ParseError("PDF has too many pages")
parts = []
extracted_chars = 0
for page in doc:
for chunk in _iter_bounded_page_text(page, fitz):
text = chunk if isinstance(chunk, str) else str(chunk)
separator_chars = 1 if parts else 0
extracted_chars += separator_chars + len(text)
if extracted_chars > Config.UPLOAD_MAX_EXTRACTED_CHARS:
raise ParseError("extracted text too large")
parts.append(text)
return "\n".join(parts)
except ParseError:
raise
except Exception as exc:
raise ParseError("could not extract PDF text") from exc
finally:
doc.close()
def parse_text(path: Path) -> str:
try:
if path.stat().st_size > Config.UPLOAD_TEXT_MAX_BYTES:
raise ParseError("text document too large")
except FileNotFoundError as exc:
raise ParseError("text document is unavailable") from exc
raw = path.read_bytes()
if len(raw) > Config.UPLOAD_TEXT_MAX_BYTES:
raise ParseError("text document too large")
# Try utf-8 first, else detect encoding
try:
return raw.decode("utf-8")
except UnicodeDecodeError:
pass
try:
from charset_normalizer import from_bytes
match = from_bytes(raw).best()
enc = getattr(match, "encoding", None) if match is not None else None
except ImportError as exc:
raise ParseError("text encoding detector is unavailable") from exc
if not enc:
raise ParseError("could not detect text encoding")
try:
return raw.decode(enc, errors="replace")
except (LookupError, UnicodeError) as exc:
raise ParseError("could not decode text document") from exc
def parse_document(path: Path) -> str:
ext = path.suffix.lower().lstrip(".")
if ext == "pdf":
return parse_pdf(path)
if ext not in {"md", "txt"}:
raise ParseError("unsupported document type")
return parse_text(path)