Spaces:
Sleeping
Sleeping
File size: 2,684 Bytes
daf58ef | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | """Parse reference/metadata files into text for LLM context.
Supports CSV, TXT, HTML, and PDF. Extracts text content that the LLM
can use to understand the study context and make better classifications.
"""
import csv
import io
from pathlib import Path
def parse_reference_file(file_path: Path) -> str:
"""Parse a reference file into plain text for LLM context.
Returns a text summary suitable for including in an LLM prompt.
Truncates to ~4000 chars to avoid blowing up the prompt.
"""
ext = file_path.suffix.lower()
try:
if ext == ".csv":
return _parse_csv(file_path)
elif ext in (".txt", ".dat", ".tsv"):
return _parse_text(file_path)
elif ext in (".html", ".htm"):
return _parse_html(file_path)
elif ext == ".pdf":
return _parse_pdf(file_path)
else:
# Try reading as plain text
return _parse_text(file_path)
except Exception as e:
return f"[Could not parse {file_path.name}: {e}]"
def _parse_csv(file_path: Path) -> str:
"""Parse CSV into a readable text table."""
lines = []
with open(file_path, newline="", encoding="utf-8", errors="replace") as f:
reader = csv.reader(f)
for i, row in enumerate(reader):
if i == 0:
lines.append("Columns: " + " | ".join(row))
elif i <= 20: # first 20 data rows
lines.append(" | ".join(row))
else:
lines.append(f"... ({i}+ more rows)")
break
return "\n".join(lines)
def _parse_text(file_path: Path) -> str:
"""Read plain text file."""
text = file_path.read_text(encoding="utf-8", errors="replace")
return text[:4000]
def _parse_html(file_path: Path) -> str:
"""Strip HTML tags and return text content."""
import re
html = file_path.read_text(encoding="utf-8", errors="replace")
# Remove script/style blocks
html = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", html, flags=re.DOTALL | re.IGNORECASE)
# Remove tags
text = re.sub(r"<[^>]+>", " ", html)
# Collapse whitespace
text = re.sub(r"\s+", " ", text).strip()
return text[:4000]
def _parse_pdf(file_path: Path) -> str:
"""Extract text from PDF. Requires pypdf (optional dependency)."""
try:
from pypdf import PdfReader
except ImportError:
return "[PDF parsing requires pypdf: pip install pypdf]"
reader = PdfReader(str(file_path))
text_parts = []
for page in reader.pages[:10]: # first 10 pages
text_parts.append(page.extract_text() or "")
text = "\n".join(text_parts)
return text[:4000]
|