Spaces:
Sleeping
Sleeping
File size: 2,320 Bytes
13779e8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | #!/usr/bin/env python3
"""Extract relevant policy text from a PDF for schema extraction.
Strategy:
1. First 3 pages (preamble + ToC + insurer/UIN).
2. Pages with high keyword density for benefits / waiting / exclusions / sub-limits.
"""
import sys
import pdfplumber
import re
path = sys.argv[1]
maxchars = int(sys.argv[2]) if len(sys.argv) > 2 else 25000
# Keyword groups with weights
PATTERNS = [
(re.compile(r"waiting period", re.I), 3),
(re.compile(r"pre[- ]?existing", re.I), 3),
(re.compile(r"sum insured", re.I), 2),
(re.compile(r"entry age|age limit|renewal", re.I), 2),
(re.compile(r"grace period|free look", re.I), 3),
(re.compile(r"room rent|icu", re.I), 3),
(re.compile(r"co[- ]?pay", re.I), 3),
(re.compile(r"deductible", re.I), 2),
(re.compile(r"day care", re.I), 2),
(re.compile(r"domiciliary|ayush|maternity|new\s*born|organ donor|ambulance", re.I), 2),
(re.compile(r"cumulative bonus|no claim|recharge|reload|restoration", re.I), 2),
(re.compile(r"network|hospitals across", re.I), 1),
(re.compile(r"critical illness", re.I), 1),
(re.compile(r"exclusion|excluded", re.I), 2),
(re.compile(r"sub[- ]?limit|cataract|knee|joint replacement", re.I), 2),
(re.compile(r"UIN", re.I), 2),
(re.compile(r"family floater|self.*spouse|dependent", re.I), 1),
(re.compile(r"₹|Rs\.|INR|lakh|crore", re.I), 1),
]
def score(text):
s = 0
for pat, w in PATTERNS:
s += len(pat.findall(text)) * w
# Penalty for "Def." dense definition pages
defs = len(re.findall(r"Def\.\s*\d+", text))
s -= defs * 2
return s
with pdfplumber.open(path) as pdf:
pages_data = []
for i, p in enumerate(pdf.pages):
t = p.extract_text() or ""
pages_data.append((i, t, score(t)))
# Always include first 3 pages
selected_idx = set([0, 1, 2])
# Sort remaining pages by score
remaining = sorted(pages_data[3:], key=lambda x: -x[2])
total = sum(len(pages_data[i][1]) for i in selected_idx)
for i, t, sc in remaining:
if total >= maxchars:
break
if sc <= 0:
continue
selected_idx.add(i)
total += len(t)
# Output in page order
out = []
for i, t, sc in pages_data:
if i in selected_idx:
out.append(f"=== PAGE {i+1} (score={sc}) ===\n{t}")
print(("\n".join(out))[:maxchars])
|