Spaces:
Sleeping
Sleeping
| """Parse reference/metadata files into text for LLM context. | |
| Supports CSV, TXT, HTML, and PDF. Extracts text content that the LLM | |
| can use to understand the study context and make better classifications. | |
| """ | |
| import csv | |
| import io | |
| from pathlib import Path | |
| def parse_reference_file(file_path: Path) -> str: | |
| """Parse a reference file into plain text for LLM context. | |
| Returns a text summary suitable for including in an LLM prompt. | |
| Truncates to ~4000 chars to avoid blowing up the prompt. | |
| """ | |
| ext = file_path.suffix.lower() | |
| try: | |
| if ext == ".csv": | |
| return _parse_csv(file_path) | |
| elif ext in (".txt", ".dat", ".tsv"): | |
| return _parse_text(file_path) | |
| elif ext in (".html", ".htm"): | |
| return _parse_html(file_path) | |
| elif ext == ".pdf": | |
| return _parse_pdf(file_path) | |
| else: | |
| # Try reading as plain text | |
| return _parse_text(file_path) | |
| except Exception as e: | |
| return f"[Could not parse {file_path.name}: {e}]" | |
| def _parse_csv(file_path: Path) -> str: | |
| """Parse CSV into a readable text table.""" | |
| lines = [] | |
| with open(file_path, newline="", encoding="utf-8", errors="replace") as f: | |
| reader = csv.reader(f) | |
| for i, row in enumerate(reader): | |
| if i == 0: | |
| lines.append("Columns: " + " | ".join(row)) | |
| elif i <= 20: # first 20 data rows | |
| lines.append(" | ".join(row)) | |
| else: | |
| lines.append(f"... ({i}+ more rows)") | |
| break | |
| return "\n".join(lines) | |
| def _parse_text(file_path: Path) -> str: | |
| """Read plain text file.""" | |
| text = file_path.read_text(encoding="utf-8", errors="replace") | |
| return text[:4000] | |
| def _parse_html(file_path: Path) -> str: | |
| """Strip HTML tags and return text content.""" | |
| import re | |
| html = file_path.read_text(encoding="utf-8", errors="replace") | |
| # Remove script/style blocks | |
| html = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", html, flags=re.DOTALL | re.IGNORECASE) | |
| # Remove tags | |
| text = re.sub(r"<[^>]+>", " ", html) | |
| # Collapse whitespace | |
| text = re.sub(r"\s+", " ", text).strip() | |
| return text[:4000] | |
| def _parse_pdf(file_path: Path) -> str: | |
| """Extract text from PDF. Requires pypdf (optional dependency).""" | |
| try: | |
| from pypdf import PdfReader | |
| except ImportError: | |
| return "[PDF parsing requires pypdf: pip install pypdf]" | |
| reader = PdfReader(str(file_path)) | |
| text_parts = [] | |
| for page in reader.pages[:10]: # first 10 pages | |
| text_parts.append(page.extract_text() or "") | |
| text = "\n".join(text_parts) | |
| return text[:4000] | |