utils.py

"""
utils.py — file/URL text extraction with thorough cleaning.
RCA fixes:
  - Regex syntax error on line 63 of old version fixed
  - Cleaning now returns proper text before chunking
"""

from __future__ import annotations
import re
import subprocess
import sys
from pathlib import Path
from typing import Tuple


def _pip_install(*pkgs):
    subprocess.check_call(
        [sys.executable, "-m", "pip", "install", "-q", "--no-warn-script-location", *pkgs],
        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
    )


def _sniff_ext(filepath: str) -> str:
    try:
        with open(filepath, "rb") as f:
            h = f.read(8)
        if h[:4] == b"%PDF":
            return ".pdf"
        if h[:4] == b"PK\x03\x04":
            return ".docx"
    except Exception:
        pass
    return ".txt"


# ---------------------------------------------------------------------------
# Main entry point
# ---------------------------------------------------------------------------

def extract_text_from_file(filepath: str) -> Tuple[str, str]:
    path = Path(filepath)
    ext  = path.suffix.lower().strip()
    name = path.stem or "document"

    if ext not in (".pdf", ".docx", ".doc", ".txt", ".md"):
        ext = _sniff_ext(filepath)

    if ext == ".pdf":
        raw = _extract_pdf(filepath)
    elif ext in (".docx", ".doc"):
        raw = _extract_docx(filepath)
    else:
        raw = path.read_text(encoding="utf-8", errors="ignore")

    text = clean_text(raw)

    if len(text) < 50:
        raise ValueError(
            f"Almost no usable text found in '{path.name}'. "
            "It may be a scanned/image-only PDF."
        )
    return text, name


# ---------------------------------------------------------------------------
# Text cleaning  (THE most important function — fixes RCA #1 and #3)
# ---------------------------------------------------------------------------

def clean_text(text: str) -> str:
    """
    Clean raw PDF/DOCX text into fluent readable prose.
    This runs BEFORE chunking so the LLM only ever sees clean text.
    """

    # 1. Fix PDF hyphenation breaks: "develop-\nment" → "development"
    text = re.sub(r"-\n\s*", "", text)

    # 2. Rejoin wrapped lines that are clearly mid-sentence
    #    e.g. "This is a long\nsentence that continues" → one line
    text = re.sub(r"(?<=[a-zA-Z,;])\n(?=[a-z])", " ", text)

    # 3. Remove known academic PDF junk lines
    junk_patterns = [
        r"this (document|material) is authorized for use only",
        r"this material has been prepared",
        r"no part of this material",
        r"reproduced.*?without.*?permission",
        r"transmitted in any form",
        r"meant for use only",
        r"all rights reserved",
        r"©|copyright\s*\d{4}",
        r"^\s*page\s*\d+\s*$",
        r"^\s*\d+\s*$",
        r"hbs no\.\s*[\d\-]+",
        r"harvard business school",
        r"s p jain|spjimr",
        r"prof\.\s+\w+.*?©",
        r"from\s+\w+\s+\d{4}\s+to\s+\w+\s+\d{4}",
    ]
    lines = text.splitlines()
    clean_lines = []
    for line in lines:
        stripped = line.strip()
        if not stripped:
            clean_lines.append("")
            continue
        if any(re.search(p, stripped, re.IGNORECASE) for p in junk_patterns):
            continue
        clean_lines.append(stripped)
    text = "\n".join(clean_lines)

    # 4. Fix mid-word spaces from column extraction: "oursel ves" → "ourselves"
    #    Only fix when a space sits between two lowercase sequences with no capitals
    text = re.sub(r"(?<=[a-z])\s(?=[a-z]{1,4}(?:\s|[.!?,]))", lambda m: "", text)

    # 5. Remove ALL-CAPS lines that are just headers/titles repeated in body
    text = re.sub(r"\n[A-Z][A-Z\s:\-]{25,}\n", "\n", text)

    # 6. Collapse excessive blank lines
    text = re.sub(r"\n{3,}", "\n\n", text)

    # 7. Strip whitespace per line
    text = "\n".join(l.strip() for l in text.splitlines())

    return text.strip()


# ---------------------------------------------------------------------------
# PDF extraction
# ---------------------------------------------------------------------------

def _extract_pdf(filepath: str) -> str:
    try:
        from pypdf import PdfReader
    except ImportError:
        _pip_install("pypdf")
        from pypdf import PdfReader

    reader = PdfReader(filepath)
    pages  = []
    for page in reader.pages:
        try:
            t = page.extract_text()
            if t and t.strip():
                pages.append(t.strip())
        except Exception:
            pass

    if not pages:
        raise ValueError(
            "No text extracted from PDF. It may be a scanned image-only PDF. "
            "Please use a text-based (selectable text) PDF."
        )
    return "\n\n".join(pages)


# ---------------------------------------------------------------------------
# DOCX extraction
# ---------------------------------------------------------------------------

def _extract_docx(filepath: str) -> str:
    try:
        import docx as _docx
    except ImportError:
        _pip_install("python-docx")
        import docx as _docx

    doc   = _docx.Document(filepath)
    parts = [p.text.strip() for p in doc.paragraphs if p.text.strip()]
    for table in doc.tables:
        for row in table.rows:
            for cell in row.cells:
                if cell.text.strip():
                    parts.append(cell.text.strip())
    return "\n\n".join(parts)


# ---------------------------------------------------------------------------
# URL extraction
# ---------------------------------------------------------------------------

def extract_text_from_url(url: str) -> Tuple[str, str]:
    import requests
    from urllib.parse import urlparse

    if "arxiv.org" in url:
        url = url.replace("/pdf/", "/abs/").replace(".pdf", "")
        if not url.startswith("http"):
            url = "https://" + url

    headers = {"User-Agent": "Mozilla/5.0 (compatible; VoiceVerse/1.0)"}
    try:
        resp = requests.get(url, headers=headers, timeout=20)
        resp.raise_for_status()
    except Exception as e:
        raise ValueError(f"Could not fetch URL: {e}")

    html     = resp.text
    doc_name = _title_from_html(html) or urlparse(url).netloc
    text     = _html_to_text(html)
    text     = clean_text(text)

    if len(text.strip()) < 100:
        raise ValueError("Could not extract enough text from that URL.")
    return text, doc_name


def _title_from_html(html: str) -> str:
    m = re.search(r"<title[^>]*>(.*?)</title>", html, re.I | re.S)
    return re.sub(r"\s+", " ", m.group(1)).strip()[:80] if m else ""


def _html_to_text(html: str) -> str:
    try:
        from bs4 import BeautifulSoup
        soup = BeautifulSoup(html, "html.parser")
        for tag in soup(["script", "style", "nav", "footer", "header",
                          "aside", "form", "button", "svg"]):
            tag.decompose()
        main = (soup.find("article") or soup.find("main")
                or soup.find("body") or soup)
        lines = [l.strip() for l in main.get_text("\n").splitlines()
                 if len(l.strip()) > 15]
        return "\n\n".join(lines)
    except Exception:
        text = re.sub(r"<(script|style)[^>]*>.*?</\1>", " ", html, flags=re.S | re.I)
        text = re.sub(r"<[^>]+>", " ", text)
        return re.sub(r"\s+", " ", text).strip()


def get_short_summary(text: str, max_chars: int = 250) -> str:
    clean = re.sub(r"\s+", " ", text.strip())
    if len(clean) <= max_chars:
        return clean
    cut = clean[:max_chars].rfind(" ")
    return clean[: cut if cut > max_chars - 30 else max_chars] + "…"