import json import logging import threading from pathlib import Path import gradio as gr import uvicorn import config from core.extractor import extract_document from core.structurer import structure_extraction from core.database import save_document, get_all_documents, get_stats, init_db from utils.helpers import validate_file, build_summary_display, format_file_size from api.routes import api_app from benchmark.compare import run_benchmark logging.basicConfig( level=logging.INFO, format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", ) logger = logging.getLogger(__name__) init_db() # ── Processing pipeline ─────────────────────────────────────────────────────── def process_document(file_obj, language_hint: str = "auto"): if file_obj is None: return ( "❌ No file uploaded. Please upload a PDF or image.", "{}", "", ) try: file_path = file_obj filename = Path(file_path).name ext = Path(file_path).suffix.lower() valid, msg = validate_file(file_path) if not valid: return (f"❌ Validation failed: {msg}", "{}", "") with open(file_path, "rb") as f: file_bytes = f.read() file_size = format_file_size(len(file_bytes)) logger.info(f"Processing: {filename} ({file_size})") raw_result = extract_document(file_bytes, ext, filename) structured = structure_extraction(raw_result) saved = save_document(structured) save_status = "✅ Saved to database" if saved else "⚠️ Save failed" summary = build_summary_display(structured) summary += f"\n\n💾 **DB Status:** {save_status}" json_output = json.dumps(structured, ensure_ascii=False, indent=2) full_text = structured.get("content", {}).get("full_text", "") if not full_text.strip(): text_preview = ( "⚠️ No text extracted. " "Try uploading a clearer image or a text-based PDF." ) else: preview = full_text[:2000] lang = structured["document_analysis"]["language"] text_preview = f"**Extracted Text** ({lang}):\n\n{preview}" if len(full_text) > 2000: text_preview += f"\n\n... [+{len(full_text)-2000} more characters]" return summary, json_output, text_preview except Exception as e: logger.error(f"Processing error: {e}", exc_info=True) return ( f"❌ Processing failed: {str(e)}", "{}", "", ) def load_documents_table(): docs = get_all_documents(limit=50) if not docs: return [["No documents yet", "", "", "", "", ""]] return [ [ d.get("document_id", ""), d.get("filename", ""), d.get("document_type", ""), d.get("language", ""), d.get("total_words", 0), f"{d.get('confidence', 0):.0%}", ] for d in docs ] def load_stats(): stats = get_stats() if not stats: return "No documents processed yet." lines = [ "## 📊 Database Statistics", "", f"**Total Documents:** {stats.get('total_documents', 0)}", f"**Arabic Documents:** {stats.get('arabic_documents', 0)}", f"**English Documents:** {stats.get('english_documents', 0)}", f"**Avg Confidence:** {stats.get('average_confidence', 0):.2%}", f"**Total Words Processed:** {stats.get('total_words_processed', 0):,}", "", "**By Document Type:**", ] for doc_type, count in stats.get("by_document_type", {}).items(): lines.append(f" - {doc_type}: {count}") return "\n".join(lines) def run_benchmark_ui(): try: results = run_benchmark() s = results["summary"] return f""" ## 🏁 Benchmark Results | Metric | Arabic | English | |--------|--------|---------| | Avg Entity Score | {s['arabic_avg_score']:.0%} | {s['english_avg_score']:.0%} | | Avg Processing Time | {s['arabic_avg_time_ms']:.1f}ms | {s['english_avg_time_ms']:.1f}ms | | Documents Tested | {s['arabic_doc_count']} | {s['english_doc_count']} | *Mock benchmark using sample data. Upload real documents for live results.* """.strip() except Exception as e: return f"❌ Benchmark error: {e}" # ── Gradio UI ───────────────────────────────────────────────────────────────── def create_ui(): with gr.Blocks( title=config.APP_TITLE, theme=gr.themes.Soft(), ) as demo: gr.HTML("""
Upload Arabic / English PDF or image → Extract → Structure → Query
Tesseract OCR engine (Arabic + English) · SQLite storage · FastAPI