Spaces:
Sleeping
Sleeping
File size: 2,190 Bytes
0e39d80 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | """General document field extraction — catch-all parser for unknown types."""
from __future__ import annotations
import re
from ml_utils.ocr import OcrResult, get_full_text, get_average_confidence
EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}")
PHONE_RE = re.compile(r"(?:\+91[\s\-]?)?[6-9]\d{9}")
DATE_RE = re.compile(
r"\d{1,2}[/\-]\d{1,2}[/\-]\d{2,4}|"
r"\d{1,2}\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\s+\d{4}",
re.I,
)
KV_RE = re.compile(r"([A-Za-z\s]{2,30})\s*[:\-]\s*(.+)")
def parse_general_fields(ocr_results: list[OcrResult]) -> dict:
"""Extract whatever we can from an unknown document type."""
full_text = get_full_text(ocr_results)
fields: dict = {}
# Emails
emails = EMAIL_RE.findall(full_text)
if emails:
fields["emails"] = emails
# Phones
phones = PHONE_RE.findall(full_text)
if phones:
fields["phones"] = phones
# Dates
dates = DATE_RE.findall(full_text)
if dates:
fields["dates"] = dates
# Key-value pairs
kv_pairs: dict = {}
for line in full_text.split("\n"):
m = KV_RE.match(line.strip())
if m:
key = m.group(1).strip().lower().replace(" ", "_")
value = m.group(2).strip()
if len(key) >= 2 and len(value) >= 1:
kv_pairs[key] = value
if kv_pairs:
fields["detected_fields"] = kv_pairs
fields["full_text_preview"] = full_text[:500] if full_text else ""
return fields
def validate_general(fields: dict, ocr_results: list[OcrResult]) -> tuple[float, list[str]]:
"""Minimal validation for unknown documents."""
flags: list[str] = ["UNKNOWN_DOC_TYPE"]
checks_total = 2
checks_passed = 0
if fields.get("detected_fields"):
checks_passed += 1
if fields.get("full_text_preview") and len(fields["full_text_preview"]) > 20:
checks_passed += 1
if ocr_results:
avg_conf = get_average_confidence(ocr_results)
if avg_conf < 0.4:
flags.append("LOW_OCR_CONFIDENCE")
validation_ratio = checks_passed / max(checks_total, 1)
return validation_ratio * 20.0, flags
|