Ritvik Shrivastava commited on
Commit ·
a8aa400
1
Parent(s): 0ec38da
fix: replace gr.Server with gr.Blocks + rewrite inference for Qwen2-VL
Browse files- app.py +159 -115
- inference.py +81 -226
- requirements.txt +17 -5
app.py
CHANGED
|
@@ -1,133 +1,177 @@
|
|
| 1 |
"""
|
| 2 |
-
app.py
|
| 3 |
-
|
| 4 |
-
GharScan — Building Defect Inspector
|
| 5 |
-
HuggingFace Space entry point.
|
| 6 |
-
|
| 7 |
-
Gradio version: >= 6.14.0 (strict requirement for gr.Server)
|
| 8 |
-
Hardware: ZeroGPU (free tier on HF Spaces)
|
| 9 |
-
Model: MiniCPM-V 2.0 fine-tuned (2.8B, OpenBMB)
|
| 10 |
-
|
| 11 |
-
Architecture:
|
| 12 |
-
gr.Server() → custom HTML/CSS/JS frontend (Off-Brand badge)
|
| 13 |
-
@app.api("analyze_defect") → ZeroGPU inference endpoint
|
| 14 |
-
@app.get("/") → serves static/index.html
|
| 15 |
-
|
| 16 |
-
Watch-Out 1 (ZeroGPU Cold Start):
|
| 17 |
-
@spaces.GPU is scoped only around the inference function call.
|
| 18 |
-
Model is globally cached; device management (cpu ↔ cuda) is handled
|
| 19 |
-
inside inference.py's run_gharscan_pipeline().
|
| 20 |
"""
|
| 21 |
-
|
| 22 |
import os
|
| 23 |
-
import
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
import spaces # HF Spaces ZeroGPU decorator
|
| 27 |
-
from gradio import Server
|
| 28 |
-
from fastapi import Request
|
| 29 |
-
from fastapi.responses import HTMLResponse, JSONResponse
|
| 30 |
-
from fastapi.staticfiles import StaticFiles
|
| 31 |
from PIL import Image
|
|
|
|
| 32 |
|
| 33 |
from inference import run_gharscan_pipeline
|
| 34 |
from agent_trace import AgentTraceLogger
|
| 35 |
|
| 36 |
-
|
| 37 |
-
# ── Static paths ───────────────────────────────────────────────────────────────
|
| 38 |
-
STATIC_DIR = Path(__file__).parent / "static"
|
| 39 |
-
|
| 40 |
-
# ── Trace logger (initialized once at startup) ────────────────────────────────
|
| 41 |
trace_logger = AgentTraceLogger()
|
| 42 |
|
| 43 |
-
|
| 44 |
-
# ── App initialization ─────────────────────────────────────────────────────────
|
| 45 |
-
app = Server()
|
| 46 |
-
|
| 47 |
-
# Serve static assets (CSS, JS, icons) at /static/*
|
| 48 |
-
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
# ── Page routes ───────────────────────────────────────────────────────────────
|
| 52 |
-
@app.get("/", response_class=HTMLResponse)
|
| 53 |
-
async def homepage(request: Request):
|
| 54 |
-
"""Serve the custom GharScan frontend."""
|
| 55 |
-
html = (STATIC_DIR / "index.html").read_text(encoding="utf-8")
|
| 56 |
-
return HTMLResponse(content=html)
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
@app.get("/health")
|
| 60 |
-
async def health():
|
| 61 |
-
return {"status": "ok", "model": "gharscan-minicpm-v2-lora"}
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
# ── Core inference API ────────────────────────────────────────────────────────
|
| 65 |
-
# ── Core inference API ────────────────────────────────────────────────────────
|
| 66 |
@spaces.GPU
|
| 67 |
-
def
|
| 68 |
-
|
|
|
|
| 69 |
session = trace_logger.start_trace()
|
| 70 |
report = run_gharscan_pipeline(image, language=language, trace_session=session)
|
| 71 |
trace_logger.save_trace(session)
|
| 72 |
return report
|
| 73 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
try:
|
| 86 |
-
return _gpu_inference(image, language)
|
| 87 |
-
except Exception as e:
|
| 88 |
-
return _error_response(f"Inference failed: {e}")
|
| 89 |
-
|
| 90 |
|
| 91 |
-
# ──
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
""
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
"
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
"""
|
| 2 |
+
app.py — GharScan HuggingFace Space
|
| 3 |
+
Uses gr.Blocks (ZeroGPU-compatible). Custom CSS for Off-Brand badge.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
"""
|
|
|
|
| 5 |
import os
|
| 6 |
+
import spaces
|
| 7 |
+
import gradio as gr
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
from PIL import Image
|
| 9 |
+
from pathlib import Path
|
| 10 |
|
| 11 |
from inference import run_gharscan_pipeline
|
| 12 |
from agent_trace import AgentTraceLogger
|
| 13 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
trace_logger = AgentTraceLogger()
|
| 15 |
|
| 16 |
+
# ── ZeroGPU-compatible inference ──────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
@spaces.GPU
|
| 18 |
+
def analyze_image(image: Image.Image, language: str) -> dict:
|
| 19 |
+
if image is None:
|
| 20 |
+
return {}
|
| 21 |
session = trace_logger.start_trace()
|
| 22 |
report = run_gharscan_pipeline(image, language=language, trace_session=session)
|
| 23 |
trace_logger.save_trace(session)
|
| 24 |
return report
|
| 25 |
|
| 26 |
+
def analyze_and_render(image, language):
|
| 27 |
+
if image is None:
|
| 28 |
+
return "<p style='color:#6b7280;padding:20px'>Please upload or take a photo first.</p>"
|
| 29 |
+
r = analyze_image(image, language)
|
| 30 |
+
if not r.get("analysis_ok"):
|
| 31 |
+
return f"<p style='color:#ef4444;padding:20px'>⚠️ {r.get('description','Analysis failed.')}</p>"
|
| 32 |
+
|
| 33 |
+
color = r.get("severity_color", "#6b7280")
|
| 34 |
+
sev = r.get("severity", 0)
|
| 35 |
+
pct = sev * 20
|
| 36 |
+
|
| 37 |
+
struct_html = ""
|
| 38 |
+
if r.get("is_structural"):
|
| 39 |
+
struct_html = f"""<div style='background:rgba(127,29,29,0.25);border:1px solid #7f1d1d;border-radius:8px;padding:12px;margin-bottom:12px;color:#fca5a5'>
|
| 40 |
+
⚠️ <strong>STRUCTURAL RISK</strong> — {r.get("structural_reasoning","")}
|
| 41 |
+
</div>"""
|
| 42 |
+
else:
|
| 43 |
+
struct_html = "<div style='background:rgba(21,128,61,0.2);border:1px solid #166534;border-radius:8px;padding:10px;margin-bottom:12px;color:#86efac'>✅ Not Structural — No immediate safety risk</div>"
|
| 44 |
+
|
| 45 |
+
liability = ""
|
| 46 |
+
if r.get("show_liability_banner"):
|
| 47 |
+
liability = f"<div style='background:rgba(239,68,68,0.1);border:1px solid rgba(239,68,68,0.3);border-radius:8px;padding:12px;color:#fca5a5;font-size:12px;margin-top:10px'>⚠️ {r.get('liability_text','')}</div>"
|
| 48 |
+
|
| 49 |
+
disclaimer = ""
|
| 50 |
+
if r.get("disclaimer"):
|
| 51 |
+
disclaimer = f"<div style='border-left:3px solid #f59e0b;padding:10px 14px;font-size:12px;color:#9ca3af;margin-top:8px'>{r['disclaimer']}</div>"
|
| 52 |
+
|
| 53 |
+
monsoon = ""
|
| 54 |
+
if r.get("monsoon_risk"):
|
| 55 |
+
monsoon = "<div style='background:rgba(234,179,8,0.1);border:1px solid rgba(234,179,8,0.3);border-radius:6px;padding:10px;color:#fde68a;font-size:13px;margin-top:10px'>🌧️ <strong>Monsoon Risk:</strong> This defect worsens during heavy rainfall. Address before June.</div>"
|
| 56 |
+
|
| 57 |
+
return f"""
|
| 58 |
+
<div style='background:#181b20;border:1px solid #2a2f38;border-radius:14px;padding:20px;font-family:Inter,sans-serif;color:#e8eaed;max-width:600px'>
|
| 59 |
+
{struct_html}
|
| 60 |
+
<div style='display:flex;justify-content:space-between;align-items:flex-start;margin-bottom:14px'>
|
| 61 |
+
<div>
|
| 62 |
+
<div style='font-size:18px;font-weight:700'>{r.get("defect_display","")}</div>
|
| 63 |
+
<div style='font-size:11px;color:#8b9099;margin-top:3px'>{r.get("defect_type","").replace("_"," ").upper()}</div>
|
| 64 |
+
</div>
|
| 65 |
+
<div style='background:#20242b;border:2px solid {color};border-radius:10px;padding:8px 14px;text-align:center'>
|
| 66 |
+
<div style='font-size:22px;font-weight:700;color:{color}'>{sev}</div>
|
| 67 |
+
<div style='font-size:10px;color:#8b9099'>{r.get("severity_label","").upper()}</div>
|
| 68 |
+
</div>
|
| 69 |
+
</div>
|
| 70 |
+
<div style='background:#20242b;border-radius:4px;height:8px;margin-bottom:4px'>
|
| 71 |
+
<div style='width:{pct}%;height:8px;border-radius:4px;background:{color}'></div>
|
| 72 |
+
</div>
|
| 73 |
+
<div style='display:flex;justify-content:space-between;font-size:10px;color:#555d6b;margin-bottom:14px'>
|
| 74 |
+
<span>Cosmetic</span><span>Moderate</span><span>Critical</span>
|
| 75 |
+
</div>
|
| 76 |
+
<div style='height:1px;background:#1e2229;margin:12px 0'></div>
|
| 77 |
+
<div style='margin-bottom:10px'>
|
| 78 |
+
<div style='font-size:10px;color:#6fb3e0;font-family:monospace;letter-spacing:.06em'>WHAT IT IS</div>
|
| 79 |
+
<div style='font-size:14px;margin-top:4px'>{r.get("description","")}</div>
|
| 80 |
+
</div>
|
| 81 |
+
<div style='margin-bottom:10px'>
|
| 82 |
+
<div style='font-size:10px;color:#6fb3e0;font-family:monospace;letter-spacing:.06em'>WHY IT HAPPENS</div>
|
| 83 |
+
<div style='font-size:14px;margin-top:4px'>{r.get("primary_cause","")}</div>
|
| 84 |
+
</div>
|
| 85 |
+
<div style='height:1px;background:#1e2229;margin:12px 0'></div>
|
| 86 |
+
<div style='background:rgba(59,130,246,0.08);border:1px solid rgba(59,130,246,0.2);border-radius:10px;padding:12px;margin-bottom:10px'>
|
| 87 |
+
<div style='font-size:10px;color:#6fb3e0;font-family:monospace'>WHAT TO DO</div>
|
| 88 |
+
<div style='font-size:14px;font-weight:500;color:#93c5fd;margin-top:4px'>{r.get("immediate_action","")}</div>
|
| 89 |
+
</div>
|
| 90 |
+
<div style='margin-bottom:12px'>
|
| 91 |
+
<div style='font-size:10px;color:#6fb3e0;font-family:monospace'>WHEN TO ACT</div>
|
| 92 |
+
<div style='font-size:14px;font-weight:500;margin-top:4px'>{r.get("urgency_display","")}</div>
|
| 93 |
+
</div>
|
| 94 |
+
<div style='height:1px;background:#1e2229;margin:12px 0'></div>
|
| 95 |
+
<div style='background:#20242b;border-radius:10px;padding:14px'>
|
| 96 |
+
<div style='font-size:22px;font-weight:700;color:#22c55e'>{r.get("cost_range_inr","")}</div>
|
| 97 |
+
<div style='font-size:13px;color:#8b9099;margin-top:6px'>👷 {r.get("professional_display","")}</div>
|
| 98 |
+
</div>
|
| 99 |
+
{monsoon}
|
| 100 |
+
{liability}
|
| 101 |
+
{disclaimer}
|
| 102 |
+
</div>
|
| 103 |
+
"""
|
| 104 |
|
| 105 |
+
# ── Custom CSS ─────────────────────────────────────────────────────────────────
|
| 106 |
+
CSS = """
|
| 107 |
+
body, .gradio-container { background: #0f1114 !important; color: #e8eaed !important; }
|
| 108 |
+
.gradio-container { max-width: 700px !important; margin: 0 auto !important; }
|
| 109 |
+
.gr-button-primary { background: linear-gradient(135deg,#2563eb,#1d4ed8) !important; border: none !important; }
|
| 110 |
+
.gr-button-primary:hover { opacity: 0.9 !important; }
|
| 111 |
+
footer { display: none !important; }
|
| 112 |
+
#component-0 { padding: 20px !important; }
|
| 113 |
+
.dark { --background-fill-primary: #181b20; --background-fill-secondary: #20242b; --border-color-primary: #2a2f38; --color-text-body: #e8eaed; }
|
| 114 |
+
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
|
| 116 |
+
# ── UI ──────────────────────────────────────────────────────────────────────────
|
| 117 |
+
with gr.Blocks(
|
| 118 |
+
css=CSS,
|
| 119 |
+
title="GharScan — Building Defect Inspector",
|
| 120 |
+
theme=gr.themes.Base(
|
| 121 |
+
primary_hue="blue",
|
| 122 |
+
neutral_hue="slate",
|
| 123 |
+
)
|
| 124 |
+
) as demo:
|
| 125 |
+
gr.HTML("""
|
| 126 |
+
<div style='display:flex;align-items:center;justify-content:space-between;
|
| 127 |
+
padding:14px 0;border-bottom:1px solid #2a2f38;margin-bottom:20px'>
|
| 128 |
+
<div style='display:flex;align-items:center;gap:10px'>
|
| 129 |
+
<span style='font-size:26px'>🏗️</span>
|
| 130 |
+
<div>
|
| 131 |
+
<div style='font-size:18px;font-weight:700;color:#e8eaed'>GharScan</div>
|
| 132 |
+
<div style='font-size:11px;color:#8b9099'>AI Building Defect Inspector · India</div>
|
| 133 |
+
</div>
|
| 134 |
+
</div>
|
| 135 |
+
<div style='background:#20242b;border:1px solid #2a2f38;border-radius:20px;padding:4px 12px;
|
| 136 |
+
font-size:11px;color:#6fb3e0;font-family:monospace'>
|
| 137 |
+
● Qwen2-VL-2B · 2.07B
|
| 138 |
+
</div>
|
| 139 |
+
</div>
|
| 140 |
+
""")
|
| 141 |
+
|
| 142 |
+
with gr.Row():
|
| 143 |
+
image_input = gr.Image(
|
| 144 |
+
sources=["upload", "webcam"],
|
| 145 |
+
type="pil",
|
| 146 |
+
label="📸 Take Photo or Upload",
|
| 147 |
+
height=300,
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
language = gr.Radio(
|
| 151 |
+
choices=["en", "hi"],
|
| 152 |
+
value="en",
|
| 153 |
+
label="Output language",
|
| 154 |
+
)
|
| 155 |
+
|
| 156 |
+
analyze_btn = gr.Button("🔍 Analyse Defect", variant="primary", size="lg")
|
| 157 |
+
report_output = gr.HTML(label="Inspection Report")
|
| 158 |
+
|
| 159 |
+
analyze_btn.click(
|
| 160 |
+
fn=analyze_and_render,
|
| 161 |
+
inputs=[image_input, language],
|
| 162 |
+
outputs=report_output,
|
| 163 |
+
api_name="analyze"
|
| 164 |
+
)
|
| 165 |
+
|
| 166 |
+
gr.HTML("""
|
| 167 |
+
<div style='text-align:center;padding:20px 0;font-size:11px;color:#555d6b;
|
| 168 |
+
border-top:1px solid #1e2229;margin-top:20px'>
|
| 169 |
+
<p>Qwen2-VL-2B fine-tuned on Indian building defects · No cloud APIs</p>
|
| 170 |
+
<p>🏗️ Built for <a href="https://huggingface.co/build-small-hackathon"
|
| 171 |
+
style='color:#8b9099'>Build Small Hackathon 2026</a> · Backyard AI Track</p>
|
| 172 |
+
<p style='color:#374151'>GharScan is a triage aid, not a substitute for professional structural assessment.</p>
|
| 173 |
+
</div>
|
| 174 |
+
""")
|
| 175 |
+
|
| 176 |
+
demo.queue()
|
| 177 |
+
demo.launch()
|
inference.py
CHANGED
|
@@ -1,276 +1,131 @@
|
|
| 1 |
"""
|
| 2 |
-
inference.py
|
| 3 |
-
────────────
|
| 4 |
-
GharScan VLM inference pipeline.
|
| 5 |
-
|
| 6 |
-
Model: MiniCPM-V 2.0 (2.8B) — fine-tuned LoRA adapter
|
| 7 |
-
Engine: HuggingFace Transformers + ZeroGPU (HF Spaces)
|
| 8 |
-
|
| 9 |
-
Watch-Out 1 (ZeroGPU Cold Start):
|
| 10 |
-
Model is loaded GLOBALLY at module level (CPU).
|
| 11 |
-
Inside @spaces.GPU, model.to("cuda") is called — GPU allocation is
|
| 12 |
-
only requested when actually needed. After inference, model.to("cpu")
|
| 13 |
-
frees the ZeroGPU allocation back to the pool.
|
| 14 |
"""
|
| 15 |
-
|
| 16 |
-
import re
|
| 17 |
-
import json
|
| 18 |
-
import time
|
| 19 |
-
import torch
|
| 20 |
from PIL import Image
|
| 21 |
from loguru import logger
|
| 22 |
from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
|
|
|
|
| 23 |
from cost_matrix import build_cost_response
|
| 24 |
|
| 25 |
-
|
| 26 |
-
LORA_MODEL_ID
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
TEMPERATURE = 0.05 # Near-greedy for structured JSON output
|
| 30 |
|
| 31 |
-
|
| 32 |
-
# ── Global model references (loaded once, stays on CPU between calls) ─────────
|
| 33 |
_model = None
|
| 34 |
-
|
| 35 |
-
|
| 36 |
|
| 37 |
def _load_model_if_needed():
|
| 38 |
-
|
| 39 |
-
Lazy model loading. Called inside @spaces.GPU context.
|
| 40 |
-
First call: loads from HF Hub (~30s on cold boot).
|
| 41 |
-
Subsequent calls: instant (already in memory).
|
| 42 |
-
"""
|
| 43 |
-
global _model, _tokenizer
|
| 44 |
-
|
| 45 |
if _model is not None:
|
| 46 |
return
|
| 47 |
-
|
| 48 |
-
logger.info(f"Loading GharScan model from {LORA_MODEL_ID} …")
|
| 49 |
t0 = time.monotonic()
|
| 50 |
-
|
| 51 |
try:
|
| 52 |
-
# Try fine-tuned LoRA model first
|
| 53 |
from peft import PeftModel
|
| 54 |
-
base =
|
| 55 |
-
BASE_MODEL_ID,
|
| 56 |
-
trust_remote_code=True,
|
| 57 |
-
torch_dtype=torch.bfloat16,
|
| 58 |
-
)
|
| 59 |
_model = PeftModel.from_pretrained(base, LORA_MODEL_ID)
|
| 60 |
_model = _model.merge_and_unload()
|
| 61 |
-
logger.info("
|
| 62 |
-
|
| 63 |
except Exception as e:
|
| 64 |
-
logger.warning(f"LoRA
|
| 65 |
-
_model =
|
| 66 |
-
BASE_MODEL_ID,
|
| 67 |
-
|
| 68 |
-
torch_dtype=torch.bfloat16,
|
| 69 |
-
)
|
| 70 |
-
|
| 71 |
-
_tokenizer = AutoTokenizer.from_pretrained(
|
| 72 |
-
BASE_MODEL_ID, trust_remote_code=True
|
| 73 |
-
)
|
| 74 |
_model.eval()
|
| 75 |
logger.info(f"Model ready in {time.monotonic()-t0:.1f}s")
|
| 76 |
|
| 77 |
-
|
| 78 |
-
# ── Prompts ────────────────────────────────────────────────────────────────────
|
| 79 |
-
_CLASSIFY_PROMPT = """You are GharScan, an expert Indian building inspector.
|
| 80 |
-
Analyze this image of a building defect and return ONLY valid JSON — no other text.
|
| 81 |
-
|
| 82 |
-
Required JSON schema:
|
| 83 |
-
{
|
| 84 |
-
"defect_type": "<hairline_crack|settlement_crack|structural_crack|water_seepage|efflorescence|spalling|rebar_rust|plaster_delamination|no_defect>",
|
| 85 |
-
"description": "<25-word plain English description of exactly what you see>",
|
| 86 |
-
"primary_cause": "<one sentence cause in simple language>",
|
| 87 |
-
"monsoon_risk": <true|false>,
|
| 88 |
-
"confidence": <0.0-1.0>
|
| 89 |
-
}"""
|
| 90 |
-
|
| 91 |
-
_SEVERITY_PROMPT = """You are GharScan. The defect in this image is: {defect_type}.
|
| 92 |
-
Assess the severity and return ONLY valid JSON — no other text.
|
| 93 |
-
|
| 94 |
-
Required JSON schema:
|
| 95 |
-
{{
|
| 96 |
-
"severity": <1|2|3|4|5>,
|
| 97 |
-
"is_structural": <true|false>,
|
| 98 |
-
"structural_reasoning": "<one sentence explanation>",
|
| 99 |
-
"immediate_action": "<specific actionable instruction in plain language>",
|
| 100 |
-
"urgency_timeline": "<next_renovation|within_6_months|within_1_month|this_week|immediately>",
|
| 101 |
-
"visible_width_estimate": "<hairline|<1mm|1-3mm|3-10mm|>10mm|not_applicable>"
|
| 102 |
-
}}
|
| 103 |
-
|
| 104 |
-
Severity scale:
|
| 105 |
-
1 = Cosmetic only, no action needed soon
|
| 106 |
-
2 = Minor, address at next renovation
|
| 107 |
-
3 = Moderate, fix within 6 months
|
| 108 |
-
4 = Serious, fix within 1 month, consider engineer
|
| 109 |
-
5 = Critical, structural risk, immediate attention"""
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
# ── Core inference helpers ───────────────────────────────────────────��────────
|
| 113 |
def _call_vlm(image: Image.Image, prompt: str) -> dict:
|
| 114 |
-
""
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
return _parse_json(raw_output)
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
def _parse_json(text: str) -> dict:
|
| 130 |
-
"""Robustly extract JSON from model output, even with surrounding text."""
|
| 131 |
-
text = text.strip()
|
| 132 |
-
|
| 133 |
-
# Try direct parse first
|
| 134 |
try:
|
| 135 |
-
return json.loads(
|
| 136 |
-
except
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
# Extract JSON object from surrounding text
|
| 140 |
-
match = re.search(r'\{[^{}]*(?:\{[^{}]*\}[^{}]*)?\}', text, re.DOTALL)
|
| 141 |
-
if match:
|
| 142 |
try:
|
| 143 |
-
return json.loads(
|
| 144 |
-
except
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
# Final fallback: return safe defaults
|
| 148 |
-
logger.warning(f"JSON parse failed on: {text[:200]}")
|
| 149 |
-
return {
|
| 150 |
-
"defect_type": "no_defect",
|
| 151 |
-
"description": "Could not analyze image clearly. Please retake with better lighting.",
|
| 152 |
-
"primary_cause": "Analysis inconclusive.",
|
| 153 |
-
"monsoon_risk": False,
|
| 154 |
-
"confidence": 0.0,
|
| 155 |
-
}
|
| 156 |
-
|
| 157 |
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
) -> dict:
|
| 174 |
-
"""
|
| 175 |
-
3-step agentic reasoning chain:
|
| 176 |
-
Step 1 → Defect classification
|
| 177 |
-
Step 2 → Severity + structural assessment
|
| 178 |
-
Step 3 → Cost lookup (deterministic, no model call)
|
| 179 |
|
| 180 |
-
|
| 181 |
-
"""
|
| 182 |
-
# ── Ensure model is loaded (cold-start safe) ──────────────────────────────
|
| 183 |
_load_model_if_needed()
|
| 184 |
-
|
| 185 |
-
# ── Move model to CUDA for this inference window ──────────────────────────
|
| 186 |
_model.to("cuda")
|
| 187 |
-
|
| 188 |
try:
|
| 189 |
-
image =
|
| 190 |
-
|
| 191 |
-
# ── Step 1: Classify defect ───────────────────────────────────────────
|
| 192 |
-
logger.info("Step 1: Classifying defect …")
|
| 193 |
-
classify_input = {"prompt": _CLASSIFY_PROMPT}
|
| 194 |
-
classify_output = _call_vlm(image, _CLASSIFY_PROMPT)
|
| 195 |
|
| 196 |
-
|
| 197 |
-
|
|
|
|
| 198 |
|
| 199 |
if trace_session:
|
| 200 |
-
trace_session.log_step("classify",
|
| 201 |
-
|
| 202 |
-
# ── Step 2: Severity assessment ───────────────────────────────────────
|
| 203 |
-
logger.info("Step 2: Assessing severity …")
|
| 204 |
-
severity_prompt = _SEVERITY_PROMPT.format(defect_type=defect_type)
|
| 205 |
-
severity_input = {"defect_type": defect_type, "prompt": severity_prompt}
|
| 206 |
-
severity_output = _call_vlm(image, severity_prompt)
|
| 207 |
|
| 208 |
-
|
| 209 |
-
|
|
|
|
| 210 |
|
| 211 |
if trace_session:
|
| 212 |
-
trace_session.log_step("severity",
|
| 213 |
|
| 214 |
-
#
|
| 215 |
-
|
| 216 |
-
cost_data = build_cost_response(defect_type, severity)
|
| 217 |
-
cost_input = {"defect_type": defect_type, "severity": severity}
|
| 218 |
|
| 219 |
if trace_session:
|
| 220 |
-
trace_session.log_step("cost",
|
| 221 |
|
| 222 |
-
# ── Assemble final report ─────────────────────────────────────────────
|
| 223 |
report = {
|
| 224 |
-
|
| 225 |
"defect_type": defect_type,
|
| 226 |
-
"defect_display":
|
| 227 |
-
"description":
|
| 228 |
-
"primary_cause":
|
| 229 |
-
"monsoon_risk":
|
| 230 |
-
"confidence": round(float(classify_output.get("confidence", 0.7)), 2),
|
| 231 |
-
# Severity
|
| 232 |
"severity": severity,
|
| 233 |
-
"severity_label":
|
| 234 |
-
"severity_color":
|
| 235 |
-
"is_structural":
|
| 236 |
-
"structural_reasoning":
|
| 237 |
-
"immediate_action":
|
| 238 |
-
"
|
| 239 |
-
"
|
| 240 |
-
"
|
| 241 |
-
|
| 242 |
-
"
|
| 243 |
-
"
|
| 244 |
-
"
|
| 245 |
-
"requires_engineer": cost_data["requires_engineer"],
|
| 246 |
-
"disclaimer": cost_data["disclaimer"],
|
| 247 |
-
# Liability banner (Watch-Out 2: shown for severity >= 4)
|
| 248 |
-
"show_liability_banner":cost_data["show_liability_banner"],
|
| 249 |
-
"liability_text": cost_data["liability_text"],
|
| 250 |
-
# Meta
|
| 251 |
-
"analysis_ok": defect_type != "no_defect",
|
| 252 |
}
|
| 253 |
-
|
| 254 |
if trace_session:
|
| 255 |
trace_session.finalize(report)
|
| 256 |
-
|
| 257 |
return report
|
| 258 |
|
| 259 |
finally:
|
| 260 |
-
# ── CRITICAL: Free ZeroGPU allocation after every call ────────────────
|
| 261 |
_model.to("cpu")
|
| 262 |
-
torch.cuda.empty_cache()
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
# ── Display name mapping ───────────────────────────────────────────────────────
|
| 266 |
-
_DEFECT_DISPLAY_NAMES = {
|
| 267 |
-
"hairline_crack": "Hairline Plaster Crack",
|
| 268 |
-
"settlement_crack": "Settlement Crack (Diagonal)",
|
| 269 |
-
"structural_crack": "Structural Crack",
|
| 270 |
-
"water_seepage": "Water Seepage / Damp Patch",
|
| 271 |
-
"efflorescence": "Efflorescence (Salt Deposits)",
|
| 272 |
-
"spalling": "Concrete Spalling",
|
| 273 |
-
"rebar_rust": "Rebar Rust Staining",
|
| 274 |
-
"plaster_delamination": "Plaster Delamination / Bubbling",
|
| 275 |
-
"no_defect": "No Defect Detected",
|
| 276 |
-
}
|
|
|
|
| 1 |
"""
|
| 2 |
+
inference.py — GharScan Qwen2-VL-2B inference pipeline
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
"""
|
| 4 |
+
import re, json, time, torch
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
from PIL import Image
|
| 6 |
from loguru import logger
|
| 7 |
from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
|
| 8 |
+
from qwen_vl_utils import process_vision_info
|
| 9 |
from cost_matrix import build_cost_response
|
| 10 |
|
| 11 |
+
BASE_MODEL_ID = "Qwen/Qwen2-VL-2B-Instruct"
|
| 12 |
+
LORA_MODEL_ID = "ritvik360/gharscan-qwen2vl-lora"
|
| 13 |
+
MAX_NEW_TOKENS = 256
|
| 14 |
+
TEMPERATURE = 0.05
|
|
|
|
| 15 |
|
|
|
|
|
|
|
| 16 |
_model = None
|
| 17 |
+
_processor = None
|
|
|
|
| 18 |
|
| 19 |
def _load_model_if_needed():
|
| 20 |
+
global _model, _processor
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
if _model is not None:
|
| 22 |
return
|
| 23 |
+
logger.info(f"Loading {LORA_MODEL_ID} …")
|
|
|
|
| 24 |
t0 = time.monotonic()
|
|
|
|
| 25 |
try:
|
|
|
|
| 26 |
from peft import PeftModel
|
| 27 |
+
base = Qwen2VLForConditionalGeneration.from_pretrained(
|
| 28 |
+
BASE_MODEL_ID, torch_dtype=torch.bfloat16)
|
|
|
|
|
|
|
|
|
|
| 29 |
_model = PeftModel.from_pretrained(base, LORA_MODEL_ID)
|
| 30 |
_model = _model.merge_and_unload()
|
| 31 |
+
logger.info("LoRA loaded ✅")
|
|
|
|
| 32 |
except Exception as e:
|
| 33 |
+
logger.warning(f"LoRA failed ({e}) — using base model")
|
| 34 |
+
_model = Qwen2VLForConditionalGeneration.from_pretrained(
|
| 35 |
+
BASE_MODEL_ID, torch_dtype=torch.bfloat16)
|
| 36 |
+
_processor = AutoProcessor.from_pretrained(BASE_MODEL_ID)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
_model.eval()
|
| 38 |
logger.info(f"Model ready in {time.monotonic()-t0:.1f}s")
|
| 39 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
def _call_vlm(image: Image.Image, prompt: str) -> dict:
|
| 41 |
+
messages = [{"role": "user", "content": [
|
| 42 |
+
{"type": "image", "image": image},
|
| 43 |
+
{"type": "text", "text": prompt}
|
| 44 |
+
]}]
|
| 45 |
+
text = _processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
| 46 |
+
image_inputs, _ = process_vision_info(messages)
|
| 47 |
+
inputs = _processor(text=[text], images=image_inputs, return_tensors="pt").to(_model.device)
|
| 48 |
+
with torch.no_grad():
|
| 49 |
+
out = _model.generate(**inputs, max_new_tokens=MAX_NEW_TOKENS,
|
| 50 |
+
temperature=TEMPERATURE, do_sample=TEMPERATURE > 0)
|
| 51 |
+
gen = out[0][inputs["input_ids"].shape[1]:]
|
| 52 |
+
raw = _processor.decode(gen, skip_special_tokens=True).strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
try:
|
| 54 |
+
return json.loads(raw)
|
| 55 |
+
except Exception:
|
| 56 |
+
m = re.search(r'\{.*\}', raw, re.DOTALL)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
try:
|
| 58 |
+
return json.loads(m.group()) if m else {}
|
| 59 |
+
except Exception:
|
| 60 |
+
return {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
|
| 62 |
+
_CLASSIFY_PROMPT = """You are GharScan, an expert Indian building inspector.
|
| 63 |
+
Analyze this image and return ONLY valid JSON:
|
| 64 |
+
{"defect_type":"<hairline_crack|settlement_crack|structural_crack|water_seepage|efflorescence|spalling|rebar_rust|plaster_delamination|no_defect>","description":"<25-word description>","primary_cause":"<1 sentence>","monsoon_risk":<true|false>,"confidence":<0.0-1.0>}"""
|
| 65 |
+
|
| 66 |
+
_SEVERITY_PROMPT = """You are GharScan. The defect is: {defect_type}.
|
| 67 |
+
Return ONLY valid JSON:
|
| 68 |
+
{{"severity":<1|2|3|4|5>,"is_structural":<bool>,"structural_reasoning":"<1 sentence>","immediate_action":"<specific action>","urgency_timeline":"<next_renovation|within_6_months|within_1_month|this_week|immediately>"}}"""
|
| 69 |
+
|
| 70 |
+
_DEFECT_DISPLAY = {
|
| 71 |
+
"hairline_crack":"Hairline Plaster Crack","settlement_crack":"Settlement Crack",
|
| 72 |
+
"structural_crack":"Structural Crack","water_seepage":"Water Seepage / Damp Patch",
|
| 73 |
+
"efflorescence":"Efflorescence (Salt Deposits)","spalling":"Concrete Spalling",
|
| 74 |
+
"rebar_rust":"Rebar Rust Staining","plaster_delamination":"Plaster Delamination",
|
| 75 |
+
"no_defect":"No Defect Detected",
|
| 76 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
|
| 78 |
+
def run_gharscan_pipeline(image: Image.Image, language: str = "en", trace_session=None) -> dict:
|
|
|
|
|
|
|
| 79 |
_load_model_if_needed()
|
|
|
|
|
|
|
| 80 |
_model.to("cuda")
|
|
|
|
| 81 |
try:
|
| 82 |
+
image = image.convert("RGB").resize((448, 448))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
|
| 84 |
+
# Step 1: Classify
|
| 85 |
+
cls = _call_vlm(image, _CLASSIFY_PROMPT)
|
| 86 |
+
defect_type = cls.get("defect_type", "no_defect")
|
| 87 |
|
| 88 |
if trace_session:
|
| 89 |
+
trace_session.log_step("classify", {}, cls)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
|
| 91 |
+
# Step 2: Severity
|
| 92 |
+
sev = _call_vlm(image, _SEVERITY_PROMPT.format(defect_type=defect_type))
|
| 93 |
+
severity = max(1, min(5, int(sev.get("severity", 2))))
|
| 94 |
|
| 95 |
if trace_session:
|
| 96 |
+
trace_session.log_step("severity", {"defect_type": defect_type}, sev)
|
| 97 |
|
| 98 |
+
# Step 3: Cost (deterministic)
|
| 99 |
+
cost = build_cost_response(defect_type, severity)
|
|
|
|
|
|
|
| 100 |
|
| 101 |
if trace_session:
|
| 102 |
+
trace_session.log_step("cost", {"defect_type": defect_type, "severity": severity}, cost)
|
| 103 |
|
|
|
|
| 104 |
report = {
|
| 105 |
+
"analysis_ok": defect_type != "no_defect",
|
| 106 |
"defect_type": defect_type,
|
| 107 |
+
"defect_display": _DEFECT_DISPLAY.get(defect_type, defect_type.replace("_"," ").title()),
|
| 108 |
+
"description": cls.get("description", ""),
|
| 109 |
+
"primary_cause": cls.get("primary_cause", ""),
|
| 110 |
+
"monsoon_risk": cls.get("monsoon_risk", False),
|
|
|
|
|
|
|
| 111 |
"severity": severity,
|
| 112 |
+
"severity_label": cost["severity_label"],
|
| 113 |
+
"severity_color": cost["severity_color"],
|
| 114 |
+
"is_structural": sev.get("is_structural", False),
|
| 115 |
+
"structural_reasoning": sev.get("structural_reasoning", ""),
|
| 116 |
+
"immediate_action": sev.get("immediate_action", ""),
|
| 117 |
+
"urgency_display": cost["urgency_display"],
|
| 118 |
+
"cost_range_inr": cost["cost_range_inr"],
|
| 119 |
+
"professional_display": cost["professional_display"],
|
| 120 |
+
"requires_engineer": cost["requires_engineer"],
|
| 121 |
+
"disclaimer": cost["disclaimer"],
|
| 122 |
+
"show_liability_banner":cost["show_liability_banner"],
|
| 123 |
+
"liability_text": cost["liability_text"],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 124 |
}
|
|
|
|
| 125 |
if trace_session:
|
| 126 |
trace_session.finalize(report)
|
|
|
|
| 127 |
return report
|
| 128 |
|
| 129 |
finally:
|
|
|
|
| 130 |
_model.to("cpu")
|
| 131 |
+
torch.cuda.empty_cache()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
requirements.txt
CHANGED
|
@@ -1,12 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
gradio>=6.14.0
|
| 2 |
transformers==4.45.2
|
| 3 |
peft==0.10.0
|
| 4 |
-
|
|
|
|
| 5 |
Pillow>=10.0.0
|
| 6 |
huggingface_hub>=0.36.0
|
| 7 |
qwen-vl-utils>=0.0.8
|
| 8 |
loguru>=0.7.0
|
| 9 |
-
spaces>=0.28.0
|
| 10 |
-
fastapi>=0.111.0
|
| 11 |
-
uvicorn>=0.29.0
|
| 12 |
-
sentencepiece==0.1.99
|
|
|
|
| 1 |
+
# gradio>=6.14.0
|
| 2 |
+
# transformers==4.45.2
|
| 3 |
+
# peft==0.10.0
|
| 4 |
+
# torch==2.11.0 # UPDATED due to configuration error
|
| 5 |
+
# Pillow>=10.0.0
|
| 6 |
+
# huggingface_hub>=0.36.0
|
| 7 |
+
# qwen-vl-utils>=0.0.8
|
| 8 |
+
# loguru>=0.7.0
|
| 9 |
+
# spaces>=0.28.0
|
| 10 |
+
# fastapi>=0.111.0
|
| 11 |
+
# uvicorn>=0.29.0
|
| 12 |
+
# sentencepiece==0.1.99
|
| 13 |
+
|
| 14 |
+
# GharScan HF Space requirements
|
| 15 |
gradio>=6.14.0
|
| 16 |
transformers==4.45.2
|
| 17 |
peft==0.10.0
|
| 18 |
+
accelerate==0.27.0
|
| 19 |
+
sentencepiece==0.1.99
|
| 20 |
Pillow>=10.0.0
|
| 21 |
huggingface_hub>=0.36.0
|
| 22 |
qwen-vl-utils>=0.0.8
|
| 23 |
loguru>=0.7.0
|
| 24 |
+
spaces>=0.28.0
|
|
|
|
|
|
|
|
|