Spaces:
Sleeping
Sleeping
File size: 3,069 Bytes
13779e8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 | #!/usr/bin/env python3
"""Extract text from all PDFs in the batch list. Write each to /tmp/pdfraw/<policy_id>.txt."""
import os
import sys
import re
import pdfplumber
BATCH_FILE = "/tmp/extract_batch_4.txt"
OUT_DIR = "/tmp/pdfraw"
os.makedirs(OUT_DIR, exist_ok=True)
MAX_CHARS = 28000
PATTERNS = [
(re.compile(r"waiting period", re.I), 3),
(re.compile(r"pre[- ]?existing", re.I), 3),
(re.compile(r"sum insured", re.I), 2),
(re.compile(r"entry age|age limit|renewal", re.I), 2),
(re.compile(r"grace period|free look", re.I), 3),
(re.compile(r"room rent|icu", re.I), 3),
(re.compile(r"co[- ]?pay", re.I), 3),
(re.compile(r"deductible", re.I), 2),
(re.compile(r"day care", re.I), 2),
(re.compile(r"domiciliary|ayush|maternity|new\s*born|organ donor|ambulance", re.I), 2),
(re.compile(r"cumulative bonus|no claim|recharge|reload|restoration", re.I), 2),
(re.compile(r"network|hospitals across", re.I), 1),
(re.compile(r"critical illness", re.I), 1),
(re.compile(r"exclusion|excluded", re.I), 2),
(re.compile(r"sub[- ]?limit|cataract|knee|joint replacement", re.I), 2),
(re.compile(r"UIN", re.I), 2),
(re.compile(r"family floater|self.*spouse|dependent", re.I), 1),
(re.compile(r"₹|Rs\.|INR|lakh|crore", re.I), 1),
]
def score(text):
s = 0
for pat, w in PATTERNS:
s += len(pat.findall(text)) * w
defs = len(re.findall(r"Def\.\s*\d+", text))
s -= defs * 2
return s
def slug_for(rel_path):
parts = rel_path.split("/")
# rag/corpus/<insurer-slug>/<file-stem>.pdf
insurer = parts[2]
stem = parts[3].replace(".pdf", "")
return f"{insurer}__{stem}"
with open(BATCH_FILE) as f:
paths = [ln.strip() for ln in f if ln.strip()]
base = "/Users/rohitsar/Documents/Personal/AI Work/Insurance Sales Bot"
for rel in paths:
pid = slug_for(rel)
out = os.path.join(OUT_DIR, pid + ".txt")
if os.path.exists(out):
continue
full = os.path.join(base, rel)
try:
with pdfplumber.open(full) as pdf:
pages_data = []
for i, p in enumerate(pdf.pages):
t = p.extract_text() or ""
pages_data.append((i, t, score(t)))
# First 3 + top-score
selected_idx = set([0, 1, 2])
remaining = sorted([(i, t, s) for i, t, s in pages_data[3:]], key=lambda x: -x[2])
total = sum(len(pages_data[i][1]) for i in selected_idx if i < len(pages_data))
for i, t, sc in remaining:
if total >= MAX_CHARS:
break
if sc <= 0:
continue
selected_idx.add(i)
total += len(t)
out_text = []
for i, t, sc in pages_data:
if i in selected_idx:
out_text.append(f"=== PAGE {i+1} (score={sc}) ===\n{t}")
result = ("\n".join(out_text))[:MAX_CHARS]
with open(out, "w") as fo:
fo.write(result)
print(f"OK {pid} pages={len(pages_data)} chars={len(result)}")
except Exception as e:
print(f"ERR {pid} {e}")
|