"""Parse reference/metadata files into text for LLM context. Supports CSV, TXT, HTML, and PDF. Extracts text content that the LLM can use to understand the study context and make better classifications. """ import csv import io from pathlib import Path def parse_reference_file(file_path: Path) -> str: """Parse a reference file into plain text for LLM context. Returns a text summary suitable for including in an LLM prompt. Truncates to ~4000 chars to avoid blowing up the prompt. """ ext = file_path.suffix.lower() try: if ext == ".csv": return _parse_csv(file_path) elif ext in (".txt", ".dat", ".tsv"): return _parse_text(file_path) elif ext in (".html", ".htm"): return _parse_html(file_path) elif ext == ".pdf": return _parse_pdf(file_path) else: # Try reading as plain text return _parse_text(file_path) except Exception as e: return f"[Could not parse {file_path.name}: {e}]" def _parse_csv(file_path: Path) -> str: """Parse CSV into a readable text table.""" lines = [] with open(file_path, newline="", encoding="utf-8", errors="replace") as f: reader = csv.reader(f) for i, row in enumerate(reader): if i == 0: lines.append("Columns: " + " | ".join(row)) elif i <= 20: # first 20 data rows lines.append(" | ".join(row)) else: lines.append(f"... ({i}+ more rows)") break return "\n".join(lines) def _parse_text(file_path: Path) -> str: """Read plain text file.""" text = file_path.read_text(encoding="utf-8", errors="replace") return text[:4000] def _parse_html(file_path: Path) -> str: """Strip HTML tags and return text content.""" import re html = file_path.read_text(encoding="utf-8", errors="replace") # Remove script/style blocks html = re.sub(r"<(script|style)[^>]*>.*?", "", html, flags=re.DOTALL | re.IGNORECASE) # Remove tags text = re.sub(r"<[^>]+>", " ", html) # Collapse whitespace text = re.sub(r"\s+", " ", text).strip() return text[:4000] def _parse_pdf(file_path: Path) -> str: """Extract text from PDF. Requires pypdf (optional dependency).""" try: from pypdf import PdfReader except ImportError: return "[PDF parsing requires pypdf: pip install pypdf]" reader = PdfReader(str(file_path)) text_parts = [] for page in reader.pages[:10]: # first 10 pages text_parts.append(page.extract_text() or "") text = "\n".join(text_parts) return text[:4000]