""" Multimodal Document Extraction Pipeline — Professional Demo Author: Aravind Kumar Nalukurthi """ import gradio as gr import plotly.graph_objects as go import os try: from pipeline.extractor import GPT4VExtractor, DOCUMENT_SCHEMAS, encode_pil_image from pipeline.validator import DocumentValidator, HumanReviewQueue PIPELINE_AVAILABLE = True except Exception as e: GPT4VExtractor = None PIPELINE_AVAILABLE = False OPENAI_KEY = os.getenv("OPENAI_API_KEY", "") CSS = """ * { box-sizing: border-box; } body, .gradio-container { background: #000 !important; font-family: -apple-system, BlinkMacSystemFont, 'SF Pro Display', 'Segoe UI', sans-serif !important; color: #f5f5f7 !important; } .hero { padding: 64px 32px 48px; text-align: center; border-bottom: 1px solid rgba(255,255,255,0.07); } .hero-badge { display: inline-block; background: rgba(255,159,10,0.12); color: #ff9f0a; font-size: 11px; font-weight: 600; letter-spacing: 0.1em; text-transform: uppercase; padding: 5px 14px; border-radius: 20px; border: 1px solid rgba(255,159,10,0.2); margin-bottom: 22px; } .hero-title { font-size: 48px; font-weight: 700; color: #f5f5f7; line-height: 1.06; letter-spacing: -0.025em; margin: 0 0 18px; } .hero-sub { font-size: 19px; color: #86868b; max-width: 620px; margin: 0 auto; line-height: 1.55; } .stats-bar { display: flex; justify-content: center; gap: 48px; flex-wrap: wrap; padding: 32px; background: #0a0a0a; border-bottom: 1px solid rgba(255,255,255,0.07); } .stat { text-align: center; } .stat-val { font-size: 30px; font-weight: 700; color: #ff9f0a; letter-spacing: -0.02em; } .stat-label { font-size: 12px; color: #6e6e73; margin-top: 3px; font-weight: 500; } .section { padding: 36px 32px; border-bottom: 1px solid rgba(255,255,255,0.06); } .sec-label { font-size: 12px; font-weight: 600; color: #6e6e73; letter-spacing: 0.09em; text-transform: uppercase; margin: 0 0 18px; } .card { background: #111; border: 1px solid rgba(255,255,255,0.08); border-radius: 14px; padding: 22px 24px; margin-bottom: 10px; } .card-title { font-size: 16px; font-weight: 600; color: #f5f5f7; margin: 0 0 8px; } .card-body { font-size: 14px; color: #86868b; line-height: 1.6; margin: 0; } .field-row { display: flex; justify-content: space-between; align-items: center; padding: 10px 0; border-bottom: 1px solid rgba(255,255,255,0.05); } .field-name { font-size: 13px; color: #86868b; font-weight: 500; } .field-value { font-size: 14px; color: #f5f5f7; font-weight: 500; } .conf-bar { height: 4px; border-radius: 2px; margin-top: 3px; } .badge-auto { background: rgba(48,209,88,0.12); color: #30d158; border: 1px solid rgba(48,209,88,0.2); padding: 4px 12px; border-radius: 8px; font-size: 12px; font-weight: 600; } .badge-review { background: rgba(255,69,58,0.12); color: #ff453a; border: 1px solid rgba(255,69,58,0.2); padding: 4px 12px; border-radius: 8px; font-size: 12px; font-weight: 600; } footer { display: none !important; } """ INVOICE_DEMO = { "fields": {"invoice_number": "INV-2024-0847", "vendor_name": "Acme Solutions LLC", "date": "2024-03-15", "subtotal": 2850.00, "tax_amount": 256.50, "total_amount": 3106.50, "currency": "USD"}, "confidence": {"invoice_number": 0.97, "vendor_name": 0.94, "date": 0.99, "subtotal": 0.91, "tax_amount": 0.88, "total_amount": 0.95, "currency": 0.99}, "overall": 0.94, "review": False, } LOW_CONF_DEMO = { "fields": {"invoice_number": "unclear", "vendor_name": "??", "date": "03/?/24", "subtotal": None, "tax_amount": None, "total_amount": 450.00, "currency": "USD"}, "confidence": {"invoice_number": 0.22, "vendor_name": 0.14, "date": 0.31, "subtotal": 0.08, "tax_amount": 0.12, "total_amount": 0.55, "currency": 0.71}, "overall": 0.19, "review": True, } def conf_color(v): return "#30d158" if v >= 0.85 else ("#ff9f0a" if v >= 0.60 else "#ff453a") def build_conf_chart(conf_dict): fields, values = list(conf_dict.keys()), [conf_dict[f] for f in conf_dict] fig = go.Figure([go.Bar( x=values, y=fields, orientation="h", marker_color=[conf_color(v) for v in values], text=[f"{v*100:.0f}%" for v in values], textposition="inside", textfont=dict(color="#000", size=12), )]) fig.update_layout( template="plotly_dark", paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", font=dict(color="#86868b"), xaxis=dict(range=[0, 1.1], gridcolor="rgba(255,255,255,0.05)"), height=280, margin=dict(t=20, b=20, l=110, r=30), showlegend=False, ) return fig def render_demo(d): badge = 'Auto-Approved' if not d["review"] else 'Sent to Human Review' rows = "".join([ f'
{k.replace("_"," ").title()}
' f'
' f'
{v if v is not None else "Not found"}
' for k, v in d["fields"].items() ]) oc = conf_color(d["overall"]) # Inline confidence chart as HTML bars (avoids gr.Plot schema bug in this Gradio version) chart_rows = "".join([ f'
' f'
{k.replace("_"," ").title()}
' f'
' f'
' f'
{d["confidence"][k]*100:.0f}%
' for k in d["confidence"] ]) return f"""
Extraction Result
{d["overall"]*100:.0f}% confident
{badge}
{rows}
Threshold: 70% — below this, document goes to human review
Per-field confidence
{chart_rows}
""" def build_conf_html(conf_dict): """Inline HTML confidence bars — avoids gr.Plot schema bug in this Gradio version.""" rows = "".join([ f'
' f'
{k.replace("_"," ").title()}
' f'
' f'
' f'
{conf_dict[k]*100:.0f}%
' for k in conf_dict ]) return f'
Per-field confidence
{rows}
' def run_extraction(image, doc_type, api_key): def _wrap(html): return gr.update(value=html) if not PIPELINE_AVAILABLE: return _wrap("

Pipeline modules not available in this environment.

") if not api_key: return _wrap("

Enter your OpenAI API key above to run live extraction.

") if image is None: return _wrap("

Upload a document image first.

") try: extractor = GPT4VExtractor(api_key=api_key) result = extractor.extract_from_image(image, doc_type) validator = DocumentValidator() val_result = validator.validate(result.extracted_data, doc_type) queue = HumanReviewQueue(confidence_threshold=0.70) queue.process(result, val_result) review_needed = result.needs_human_review or val_result.has_errors badge = 'Sent to Human Review' if review_needed else 'Auto-Approved' oc = conf_color(result.overall_confidence) rows = "".join([ f'
{k.replace("_"," ").title()}
' f'
' f'
{v}
' for k, v in result.extracted_data.items() ]) html = ( f'
' f'
' f'
Live Extraction - {doc_type.title()}
' f'
{result.overall_confidence*100:.0f}% confident
' f'{badge}
{rows}
' + build_conf_html(result.confidence_scores) ) return _wrap(html) except Exception as e: return _wrap(f"

Error: {e}

") def _show_good(): return gr.update(value=render_demo(INVOICE_DEMO)) def _show_bad(): return gr.update(value=render_demo(LOW_CONF_DEMO)) with gr.Blocks(css=CSS, theme=gr.themes.Base(), title="Multimodal Document Extraction") as demo: gr.HTML("""
AI Engineering · Vision + Extraction

Multimodal Document Extraction

Upload any document image and GPT-4o reads it, extracts every field into structured JSON, validates the data for consistency, and routes anything uncertain to a human review queue — automatically.

5
Document types supported
94%
Confidence on clean docs
70%
Auto-approval threshold
1
API key required (OpenAI)
""") with gr.Tabs(): with gr.Tab("Overview"): gr.HTML("""
How it works
The problem

Businesses process thousands of documents daily — invoices, contracts, IDs. Manual data entry is slow and error-prone. Traditional OCR extracts raw text but doesn't understand structure or meaning. This pipeline uses GPT-4o's vision to understand documents the way a human would.

Step 1 — GPT-4o Extraction

The image is sent to GPT-4o with a schema-driven prompt ("Extract these specific fields from this invoice"). The model returns structured JSON with per-field confidence scores. High-detail mode is used for small text.

Step 2 — Validation

The extracted JSON is checked for consistency. For invoices: does subtotal + tax ≈ total (within 2%)? Are dates valid? Are required fields present? Each error adds a confidence penalty.

Step 3 — Human Review Queue

Documents above 70% confidence are auto-approved. Below 70%: queued for human review. Below 40%: HIGH priority review. This lets human effort focus on the 10% of documents that actually need it.

How to explore

No API key: Click "Pre-computed Examples" to see results on a clean invoice and a low-quality scan.
With API key: Go to "Live Extraction" and upload any document.

""") with gr.Tab("Pre-computed Examples"): gr.HTML('
No API key needed
') with gr.Row(): btn_good = gr.Button("Clean invoice — 94% confidence", size="sm") btn_bad = gr.Button("Blurry scan — 19% confidence", size="sm") demo_html = gr.HTML(value="") btn_good.click(fn=_show_good, outputs=[demo_html]) btn_bad.click(fn=_show_bad, outputs=[demo_html]) with gr.Tab("Live Extraction"): gr.HTML('
Requires OpenAI API key
') api_key = gr.Textbox(label="OpenAI API Key", type="password", value=OPENAI_KEY) with gr.Row(): img_upload = gr.Image(label="Document Image", type="pil") doc_type = gr.Dropdown(choices=["invoice", "id_document", "medical_record", "business_card", "form"], value="invoice", label="Document Type") extract_btn = gr.Button("Extract & Validate", variant="primary") live_html = gr.HTML(value="") extract_btn.click(fn=run_extraction, inputs=[img_upload, doc_type, api_key], outputs=[live_html]) with gr.Tab("How It Works"): gr.Markdown(""" ## 3-Stage Pipeline ``` Image → GPT-4o Vision → Schema Validation → Human Review Queue ``` ## Extraction (GPT-4o) ```python response = client.chat.completions.create( model="gpt-4o", messages=[{"role": "user", "content": [ {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}", "detail": "high"}}, {"type": "text", "text": f"Extract fields. Schema: {schema}. Return confidence_scores dict (0-1 per field)."}, ]}], response_format={"type": "json_object"}, ) ``` ## Validation (Invoice example) ```python # Cross-field consistency check if abs((subtotal + tax) - total) / total > 0.02: errors.append("subtotal + tax != total") confidence_penalty += 0.15 ``` ## Human Review Logic ```python final_confidence = extraction_confidence - validation_penalty if final_confidence < 0.70: queue.add(document, priority="HIGH" if final_confidence < 0.40 else "NORMAL") else: auto_approve(document) ``` ## Supported Documents - **Invoice** — vendor, date, line items, totals, tax - **ID Document** — name, DOB, ID number, expiry, address - **Medical Record** — patient, diagnosis, medications, vitals - **Business Card** — name, title, company, contact info - **Generic Form** — dynamic field detection """) demo.launch()