Spaces:
Build error
Build error
| """ | |
| Benchmark: Arabic vs English Document Extraction | |
| Compares extraction quality, speed, and accuracy between languages. | |
| """ | |
| import time | |
| import re | |
| import json | |
| from datetime import datetime | |
| # ── Sample texts ────────────────────────────────────────────────────────────── | |
| ARABIC_SAMPLES = [ | |
| { | |
| "name": "Invoice (Arabic)", | |
| "text": """ | |
| فاتورة رقم: ١٢٣٤٥ | |
| التاريخ: ١٥ يناير ٢٠٢٤ | |
| اسم العميل: شركة النور للتجارة | |
| العنوان: الرياض، المملكة العربية السعودية | |
| البريد الإلكتروني: info@alnour.com | |
| رقم الهاتف: +966501234567 | |
| المنتجات: | |
| - برنامج إدارة المخزون: ٥٠٠٠ ريال | |
| - خدمة الدعم السنوي: ١٢٠٠ ريال | |
| - التدريب والتأهيل: ٢٠٠٠ ريال | |
| الإجمالي: ٨٢٠٠ ريال سعودي | |
| ضريبة القيمة المضافة (١٥٪): ١٢٣٠ ريال | |
| المجموع الكلي: ٩٤٣٠ ريال سعودي | |
| """, | |
| "expected_entities": ["email", "phone", "amount", "date"] | |
| }, | |
| { | |
| "name": "Contract (Arabic)", | |
| "text": """ | |
| عقد خدمات تقنية المعلومات | |
| رقم العقد: IT-2024-007 | |
| تاريخ الإبرام: ٠١/٠٣/٢٠٢٤ | |
| الطرف الأول: شركة التقنية المتقدمة المحدودة | |
| الطرف الثاني: مؤسسة الأعمال الرقمية | |
| مدة العقد: اثنا عشر شهراً | |
| قيمة العقد: ١٥٠,٠٠٠ ريال سعودي | |
| البنود والشروط: | |
| ١. يلتزم الطرف الأول بتقديم خدمات الدعم التقني على مدار الساعة. | |
| ٢. يلتزم الطرف الثاني بالسداد وفقاً للجدول الزمني المتفق عليه. | |
| ٣. في حالة الإخلال بالعقد تُطبق غرامات التأخير المنصوص عليها. | |
| """, | |
| "expected_entities": ["date", "amount", "organization"] | |
| }, | |
| { | |
| "name": "Report (Arabic)", | |
| "text": """ | |
| تقرير الأداء المالي - الربع الأول ٢٠٢٤ | |
| ملخص تنفيذي: | |
| حققت الشركة نمواً ملحوظاً في الربع الأول من عام ٢٠٢٤، | |
| إذ بلغت الإيرادات الإجمالية ٢.٥ مليون ريال سعودي. | |
| المؤشرات الرئيسية: | |
| - نمو الإيرادات: ٢٣٪ مقارنة بالفترة ذاتها من العام الماضي | |
| - هامش الربح الصافي: ١٨٪ | |
| - عدد العملاء الجدد: ١٤٧ عميل | |
| - معدل الاحتفاظ بالعملاء: ٩٢٪ | |
| التوصيات: | |
| يوصي الفريق بزيادة الاستثمار في قطاع التحول الرقمي | |
| والتوسع في الأسواق الخليجية خلال النصف الثاني من العام. | |
| """, | |
| "expected_entities": ["date", "amount", "percentage"] | |
| } | |
| ] | |
| ENGLISH_SAMPLES = [ | |
| { | |
| "name": "Invoice (English)", | |
| "text": """ | |
| Invoice Number: 12345 | |
| Date: January 15, 2024 | |
| Client: Al Nour Trading Company | |
| Address: Riyadh, Saudi Arabia | |
| Email: info@alnour.com | |
| Phone: +966501234567 | |
| Items: | |
| - Inventory Management Software: $5,000 | |
| - Annual Support Service: $1,200 | |
| - Training and Onboarding: $2,000 | |
| Subtotal: $8,200 | |
| VAT (15%): $1,230 | |
| Total: $9,430 USD | |
| """, | |
| "expected_entities": ["email", "phone", "amount", "date"] | |
| }, | |
| { | |
| "name": "Contract (English)", | |
| "text": """ | |
| IT Services Agreement | |
| Contract Number: IT-2024-007 | |
| Date: 01/03/2024 | |
| Party A: Advanced Technology Solutions LLC | |
| Party B: Digital Business Enterprise | |
| Contract Duration: Twelve (12) months | |
| Contract Value: $150,000 USD | |
| Terms and Conditions: | |
| 1. Party A shall provide round-the-clock technical support. | |
| 2. Party B shall make payments per the agreed schedule. | |
| 3. Breach of contract shall incur the penalties specified herein. | |
| """, | |
| "expected_entities": ["date", "amount", "organization"] | |
| }, | |
| { | |
| "name": "Report (English)", | |
| "text": """ | |
| Financial Performance Report - Q1 2024 | |
| Executive Summary: | |
| The company achieved remarkable growth in Q1 2024, | |
| with total revenues reaching SAR 2.5 million. | |
| Key Performance Indicators: | |
| - Revenue Growth: 23% compared to the same period last year | |
| - Net Profit Margin: 18% | |
| - New Customers Acquired: 147 | |
| - Customer Retention Rate: 92% | |
| Recommendations: | |
| The team recommends increasing investment in digital transformation | |
| and expanding into Gulf markets in H2 2024. | |
| """, | |
| "expected_entities": ["date", "amount", "percentage"] | |
| } | |
| ] | |
| # ── Benchmark functions ─────────────────────────────────────────────────────── | |
| def mock_extract(text: str, language: str = "auto") -> dict: | |
| """Simulate extraction (no real API needed for benchmark).""" | |
| start = time.time() | |
| time.sleep(0.01) # Simulate processing time | |
| arabic_chars = len(re.findall(r'[\u0600-\u06FF]', text)) | |
| total_chars = len(text) | |
| arabic_ratio = arabic_chars / max(total_chars, 1) | |
| emails = re.findall(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', text) | |
| phones = re.findall(r'\+?\d[\d\s\-]{7,15}', text) | |
| amounts_ar = re.findall(r'[\d,٠-٩]+\s*(?:ريال|درهم|دولار|جنيه)', text) | |
| amounts_en = re.findall(r'[\$€£]\s*[\d,]+|[\d,]+\s*(?:USD|EUR|SAR|AED)', text) | |
| dates_ar = re.findall(r'\d{1,2}\s+(?:يناير|فبراير|مارس|أبريل|مايو|يونيو|يوليو|أغسطس|سبتمبر|أكتوبر|نوفمبر|ديسمبر)\s+\d{4}', text) | |
| dates_en = re.findall(r'\d{1,2}[/\-]\d{1,2}[/\-]\d{2,4}|\w+ \d{1,2},\s*\d{4}', text) | |
| elapsed_ms = (time.time() - start) * 1000 | |
| return { | |
| "word_count": len(text.split()), | |
| "char_count": total_chars, | |
| "arabic_char_ratio": round(arabic_ratio, 4), | |
| "emails_found": len(emails), | |
| "phones_found": len(phones), | |
| "amounts_found": len(amounts_ar) + len(amounts_en), | |
| "dates_found": len(dates_ar) + len(dates_en), | |
| "processing_time_ms": round(elapsed_ms, 2), | |
| "simulated_confidence": 0.92 if arabic_ratio < 0.5 else 0.87 | |
| } | |
| def score_extraction(result: dict, expected_entities: list) -> float: | |
| """Score extraction based on expected entity types found.""" | |
| score = 0.0 | |
| checks = { | |
| "email": result["emails_found"] > 0, | |
| "phone": result["phones_found"] > 0, | |
| "amount": result["amounts_found"] > 0, | |
| "date": result["dates_found"] > 0, | |
| "organization": result["word_count"] > 20, | |
| "percentage": result["word_count"] > 10, | |
| } | |
| for entity in expected_entities: | |
| if checks.get(entity, False): | |
| score += 1.0 | |
| return round(score / max(len(expected_entities), 1), 4) | |
| def run_benchmark() -> dict: | |
| """Run full Arabic vs English benchmark.""" | |
| print("\n" + "="*60) | |
| print(" ARABIC vs ENGLISH DOCUMENT EXTRACTION BENCHMARK") | |
| print("="*60) | |
| results = { | |
| "run_at": datetime.utcnow().isoformat(), | |
| "arabic": [], | |
| "english": [], | |
| "summary": {} | |
| } | |
| print("\n📄 ARABIC DOCUMENTS:") | |
| print("-" * 40) | |
| for sample in ARABIC_SAMPLES: | |
| res = mock_extract(sample["text"], "ar") | |
| score = score_extraction(res, sample["expected_entities"]) | |
| res["document_name"] = sample["name"] | |
| res["entity_score"] = score | |
| results["arabic"].append(res) | |
| print(f" ✓ {sample['name']}") | |
| print(f" Words: {res['word_count']} | Chars: {res['char_count']}") | |
| print(f" Arabic Ratio: {res['arabic_char_ratio']:.1%}") | |
| print(f" Entities: emails={res['emails_found']} phones={res['phones_found']} " | |
| f"amounts={res['amounts_found']} dates={res['dates_found']}") | |
| print(f" Score: {score:.0%} | Time: {res['processing_time_ms']}ms") | |
| print() | |
| print("📄 ENGLISH DOCUMENTS:") | |
| print("-" * 40) | |
| for sample in ENGLISH_SAMPLES: | |
| res = mock_extract(sample["text"], "en") | |
| score = score_extraction(res, sample["expected_entities"]) | |
| res["document_name"] = sample["name"] | |
| res["entity_score"] = score | |
| results["english"].append(res) | |
| print(f" ✓ {sample['name']}") | |
| print(f" Words: {res['word_count']} | Chars: {res['char_count']}") | |
| print(f" Arabic Ratio: {res['arabic_char_ratio']:.1%}") | |
| print(f" Entities: emails={res['emails_found']} phones={res['phones_found']} " | |
| f"amounts={res['amounts_found']} dates={res['dates_found']}") | |
| print(f" Score: {score:.0%} | Time: {res['processing_time_ms']}ms") | |
| print() | |
| # Summary | |
| ar_scores = [r["entity_score"] for r in results["arabic"]] | |
| en_scores = [r["entity_score"] for r in results["english"]] | |
| ar_times = [r["processing_time_ms"] for r in results["arabic"]] | |
| en_times = [r["processing_time_ms"] for r in results["english"]] | |
| results["summary"] = { | |
| "arabic_avg_score": round(sum(ar_scores) / len(ar_scores), 4), | |
| "english_avg_score": round(sum(en_scores) / len(en_scores), 4), | |
| "arabic_avg_time_ms": round(sum(ar_times) / len(ar_times), 2), | |
| "english_avg_time_ms": round(sum(en_times) / len(en_times), 2), | |
| "arabic_doc_count": len(results["arabic"]), | |
| "english_doc_count": len(results["english"]) | |
| } | |
| s = results["summary"] | |
| print("="*60) | |
| print(" BENCHMARK SUMMARY") | |
| print("="*60) | |
| print(f" Arabic — Avg Score: {s['arabic_avg_score']:.0%} | Avg Time: {s['arabic_avg_time_ms']:.1f}ms") | |
| print(f" English — Avg Score: {s['english_avg_score']:.0%} | Avg Time: {s['english_avg_time_ms']:.1f}ms") | |
| winner = "Arabic" if s["arabic_avg_score"] >= s["english_avg_score"] else "English" | |
| print(f"\n 🏆 Better Entity Extraction: {winner}") | |
| print("="*60) | |
| return results | |
| if __name__ == "__main__": | |
| results = run_benchmark() | |
| with open("benchmark_results.json", "w", ensure_ascii=False) as f: | |
| json.dump(results, f, indent=2, ensure_ascii=False) | |
| print("\n✅ Results saved to benchmark_results.json") |