""" Benchmark: Arabic vs English Document Extraction Compares extraction quality, speed, and accuracy between languages. """ import time import re import json from datetime import datetime # ── Sample texts ────────────────────────────────────────────────────────────── ARABIC_SAMPLES = [ { "name": "Invoice (Arabic)", "text": """ فاتورة رقم: ١٢٣٤٥ التاريخ: ١٥ يناير ٢٠٢٤ اسم العميل: شركة النور للتجارة العنوان: الرياض، المملكة العربية السعودية البريد الإلكتروني: info@alnour.com رقم الهاتف: +966501234567 المنتجات: - برنامج إدارة المخزون: ٥٠٠٠ ريال - خدمة الدعم السنوي: ١٢٠٠ ريال - التدريب والتأهيل: ٢٠٠٠ ريال الإجمالي: ٨٢٠٠ ريال سعودي ضريبة القيمة المضافة (١٥٪): ١٢٣٠ ريال المجموع الكلي: ٩٤٣٠ ريال سعودي """, "expected_entities": ["email", "phone", "amount", "date"] }, { "name": "Contract (Arabic)", "text": """ عقد خدمات تقنية المعلومات رقم العقد: IT-2024-007 تاريخ الإبرام: ٠١/٠٣/٢٠٢٤ الطرف الأول: شركة التقنية المتقدمة المحدودة الطرف الثاني: مؤسسة الأعمال الرقمية مدة العقد: اثنا عشر شهراً قيمة العقد: ١٥٠,٠٠٠ ريال سعودي البنود والشروط: ١. يلتزم الطرف الأول بتقديم خدمات الدعم التقني على مدار الساعة. ٢. يلتزم الطرف الثاني بالسداد وفقاً للجدول الزمني المتفق عليه. ٣. في حالة الإخلال بالعقد تُطبق غرامات التأخير المنصوص عليها. """, "expected_entities": ["date", "amount", "organization"] }, { "name": "Report (Arabic)", "text": """ تقرير الأداء المالي - الربع الأول ٢٠٢٤ ملخص تنفيذي: حققت الشركة نمواً ملحوظاً في الربع الأول من عام ٢٠٢٤، إذ بلغت الإيرادات الإجمالية ٢.٥ مليون ريال سعودي. المؤشرات الرئيسية: - نمو الإيرادات: ٢٣٪ مقارنة بالفترة ذاتها من العام الماضي - هامش الربح الصافي: ١٨٪ - عدد العملاء الجدد: ١٤٧ عميل - معدل الاحتفاظ بالعملاء: ٩٢٪ التوصيات: يوصي الفريق بزيادة الاستثمار في قطاع التحول الرقمي والتوسع في الأسواق الخليجية خلال النصف الثاني من العام. """, "expected_entities": ["date", "amount", "percentage"] } ] ENGLISH_SAMPLES = [ { "name": "Invoice (English)", "text": """ Invoice Number: 12345 Date: January 15, 2024 Client: Al Nour Trading Company Address: Riyadh, Saudi Arabia Email: info@alnour.com Phone: +966501234567 Items: - Inventory Management Software: $5,000 - Annual Support Service: $1,200 - Training and Onboarding: $2,000 Subtotal: $8,200 VAT (15%): $1,230 Total: $9,430 USD """, "expected_entities": ["email", "phone", "amount", "date"] }, { "name": "Contract (English)", "text": """ IT Services Agreement Contract Number: IT-2024-007 Date: 01/03/2024 Party A: Advanced Technology Solutions LLC Party B: Digital Business Enterprise Contract Duration: Twelve (12) months Contract Value: $150,000 USD Terms and Conditions: 1. Party A shall provide round-the-clock technical support. 2. Party B shall make payments per the agreed schedule. 3. Breach of contract shall incur the penalties specified herein. """, "expected_entities": ["date", "amount", "organization"] }, { "name": "Report (English)", "text": """ Financial Performance Report - Q1 2024 Executive Summary: The company achieved remarkable growth in Q1 2024, with total revenues reaching SAR 2.5 million. Key Performance Indicators: - Revenue Growth: 23% compared to the same period last year - Net Profit Margin: 18% - New Customers Acquired: 147 - Customer Retention Rate: 92% Recommendations: The team recommends increasing investment in digital transformation and expanding into Gulf markets in H2 2024. """, "expected_entities": ["date", "amount", "percentage"] } ] # ── Benchmark functions ─────────────────────────────────────────────────────── def mock_extract(text: str, language: str = "auto") -> dict: """Simulate extraction (no real API needed for benchmark).""" start = time.time() time.sleep(0.01) # Simulate processing time arabic_chars = len(re.findall(r'[\u0600-\u06FF]', text)) total_chars = len(text) arabic_ratio = arabic_chars / max(total_chars, 1) emails = re.findall(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', text) phones = re.findall(r'\+?\d[\d\s\-]{7,15}', text) amounts_ar = re.findall(r'[\d,٠-٩]+\s*(?:ريال|درهم|دولار|جنيه)', text) amounts_en = re.findall(r'[\$€£]\s*[\d,]+|[\d,]+\s*(?:USD|EUR|SAR|AED)', text) dates_ar = re.findall(r'\d{1,2}\s+(?:يناير|فبراير|مارس|أبريل|مايو|يونيو|يوليو|أغسطس|سبتمبر|أكتوبر|نوفمبر|ديسمبر)\s+\d{4}', text) dates_en = re.findall(r'\d{1,2}[/\-]\d{1,2}[/\-]\d{2,4}|\w+ \d{1,2},\s*\d{4}', text) elapsed_ms = (time.time() - start) * 1000 return { "word_count": len(text.split()), "char_count": total_chars, "arabic_char_ratio": round(arabic_ratio, 4), "emails_found": len(emails), "phones_found": len(phones), "amounts_found": len(amounts_ar) + len(amounts_en), "dates_found": len(dates_ar) + len(dates_en), "processing_time_ms": round(elapsed_ms, 2), "simulated_confidence": 0.92 if arabic_ratio < 0.5 else 0.87 } def score_extraction(result: dict, expected_entities: list) -> float: """Score extraction based on expected entity types found.""" score = 0.0 checks = { "email": result["emails_found"] > 0, "phone": result["phones_found"] > 0, "amount": result["amounts_found"] > 0, "date": result["dates_found"] > 0, "organization": result["word_count"] > 20, "percentage": result["word_count"] > 10, } for entity in expected_entities: if checks.get(entity, False): score += 1.0 return round(score / max(len(expected_entities), 1), 4) def run_benchmark() -> dict: """Run full Arabic vs English benchmark.""" print("\n" + "="*60) print(" ARABIC vs ENGLISH DOCUMENT EXTRACTION BENCHMARK") print("="*60) results = { "run_at": datetime.utcnow().isoformat(), "arabic": [], "english": [], "summary": {} } print("\n📄 ARABIC DOCUMENTS:") print("-" * 40) for sample in ARABIC_SAMPLES: res = mock_extract(sample["text"], "ar") score = score_extraction(res, sample["expected_entities"]) res["document_name"] = sample["name"] res["entity_score"] = score results["arabic"].append(res) print(f" ✓ {sample['name']}") print(f" Words: {res['word_count']} | Chars: {res['char_count']}") print(f" Arabic Ratio: {res['arabic_char_ratio']:.1%}") print(f" Entities: emails={res['emails_found']} phones={res['phones_found']} " f"amounts={res['amounts_found']} dates={res['dates_found']}") print(f" Score: {score:.0%} | Time: {res['processing_time_ms']}ms") print() print("📄 ENGLISH DOCUMENTS:") print("-" * 40) for sample in ENGLISH_SAMPLES: res = mock_extract(sample["text"], "en") score = score_extraction(res, sample["expected_entities"]) res["document_name"] = sample["name"] res["entity_score"] = score results["english"].append(res) print(f" ✓ {sample['name']}") print(f" Words: {res['word_count']} | Chars: {res['char_count']}") print(f" Arabic Ratio: {res['arabic_char_ratio']:.1%}") print(f" Entities: emails={res['emails_found']} phones={res['phones_found']} " f"amounts={res['amounts_found']} dates={res['dates_found']}") print(f" Score: {score:.0%} | Time: {res['processing_time_ms']}ms") print() # Summary ar_scores = [r["entity_score"] for r in results["arabic"]] en_scores = [r["entity_score"] for r in results["english"]] ar_times = [r["processing_time_ms"] for r in results["arabic"]] en_times = [r["processing_time_ms"] for r in results["english"]] results["summary"] = { "arabic_avg_score": round(sum(ar_scores) / len(ar_scores), 4), "english_avg_score": round(sum(en_scores) / len(en_scores), 4), "arabic_avg_time_ms": round(sum(ar_times) / len(ar_times), 2), "english_avg_time_ms": round(sum(en_times) / len(en_times), 2), "arabic_doc_count": len(results["arabic"]), "english_doc_count": len(results["english"]) } s = results["summary"] print("="*60) print(" BENCHMARK SUMMARY") print("="*60) print(f" Arabic — Avg Score: {s['arabic_avg_score']:.0%} | Avg Time: {s['arabic_avg_time_ms']:.1f}ms") print(f" English — Avg Score: {s['english_avg_score']:.0%} | Avg Time: {s['english_avg_time_ms']:.1f}ms") winner = "Arabic" if s["arabic_avg_score"] >= s["english_avg_score"] else "English" print(f"\n 🏆 Better Entity Extraction: {winner}") print("="*60) return results if __name__ == "__main__": results = run_benchmark() with open("benchmark_results.json", "w", ensure_ascii=False) as f: json.dump(results, f, indent=2, ensure_ascii=False) print("\n✅ Results saved to benchmark_results.json")