Spaces:
Running on Zero
Running on Zero
feat: integrate Cloudflare R2 Quran translations, add translation endpoints and edge caching
3c6e2f1 | """ | |
| Comprehensive verification script for Quran translation files | |
| Checks: | |
| 1. All 114 files exist | |
| 2. Correct ayah counts | |
| 3. All translations present | |
| 4. Valid JSON structure | |
| 5. No empty translations | |
| """ | |
| import json | |
| from pathlib import Path | |
| OUTPUT_DIR = Path(__file__).parent.parent / "data" / "quran_translation" | |
| # Expected translations | |
| EXPECTED_TRANSLATIONS = ["en.sahih", "ur.jalandhry", "ur.junagarhi", "hi.farooq"] | |
| # Expected ayah counts for each surah | |
| AYAH_COUNTS = [ | |
| 7, 286, 200, 176, 120, 165, 206, 75, 129, 109, | |
| 123, 111, 43, 52, 99, 128, 111, 110, 98, 135, | |
| 112, 78, 118, 64, 77, 227, 93, 88, 69, 60, | |
| 34, 30, 73, 54, 45, 83, 182, 88, 75, 85, | |
| 54, 53, 89, 59, 37, 35, 38, 29, 18, 45, | |
| 60, 49, 62, 55, 78, 96, 29, 22, 24, 13, | |
| 14, 11, 11, 18, 12, 12, 30, 52, 52, 44, | |
| 28, 28, 20, 56, 40, 31, 50, 40, 46, 42, | |
| 29, 19, 36, 25, 22, 17, 19, 26, 30, 20, | |
| 15, 21, 11, 8, 8, 19, 5, 8, 8, 11, | |
| 11, 8, 3, 9, 5, 4, 7, 3, 6, 3, | |
| 5, 4, 5, 6 | |
| ] | |
| def verify_all_files(): | |
| print("🔍 COMPREHENSIVE VERIFICATION") | |
| print("=" * 80) | |
| all_files = list(OUTPUT_DIR.glob("*_Translation.json")) | |
| # Statistics | |
| total_surahs = 114 | |
| complete_surahs = 0 | |
| incomplete_surahs = [] | |
| missing_surahs = [] | |
| corrupted_files = [] | |
| # Track issues | |
| ayah_count_issues = [] | |
| missing_translation_issues = [] | |
| empty_text_issues = [] | |
| print(f"📁 Found {len(all_files)} translation files\n") | |
| # Create a map of surah_no to file | |
| surah_files = {} | |
| for file in all_files: | |
| try: | |
| with open(file, 'r', encoding='utf-8') as f: | |
| data = json.load(f) | |
| surah_no = data['surah']['surah_no'] | |
| surah_files[surah_no] = { | |
| 'file': file, | |
| 'data': data | |
| } | |
| except Exception as e: | |
| corrupted_files.append({'file': file.name, 'error': str(e)}) | |
| print(f"❌ {file.name}: Corrupted JSON - {e}") | |
| # Check each surah | |
| for surah_no in range(1, total_surahs + 1): | |
| expected_ayahs = AYAH_COUNTS[surah_no - 1] | |
| # Check if file exists | |
| if surah_no not in surah_files: | |
| missing_surahs.append(surah_no) | |
| print(f"❌ Surah {surah_no:3d}: File missing") | |
| continue | |
| file_info = surah_files[surah_no] | |
| data = file_info['data'] | |
| surah_name = data['surah']['name_english'] | |
| actual_ayahs = len(data['ayahs']) | |
| # Initialize issue tracking for this surah | |
| has_issues = False | |
| surah_issues = [] | |
| # Check 1: Ayah count | |
| if actual_ayahs != expected_ayahs: | |
| has_issues = True | |
| issue = f"Expected {expected_ayahs} ayahs, got {actual_ayahs}" | |
| surah_issues.append(issue) | |
| ayah_count_issues.append({ | |
| 'surah_no': surah_no, | |
| 'name': surah_name, | |
| 'expected': expected_ayahs, | |
| 'actual': actual_ayahs | |
| }) | |
| # Check 2: All ayahs have all translations | |
| for ayah in data['ayahs']: | |
| ayah_no = ayah['ayah_no'] | |
| # Check if all expected translations are present | |
| for trans_key in EXPECTED_TRANSLATIONS: | |
| if trans_key not in ayah.get('translations', {}): | |
| has_issues = True | |
| issue = f"Ayah {ayah_no} missing {trans_key}" | |
| if issue not in surah_issues: | |
| surah_issues.append(issue) | |
| missing_translation_issues.append({ | |
| 'surah_no': surah_no, | |
| 'surah_name': surah_name, | |
| 'ayah_no': ayah_no, | |
| 'missing': trans_key | |
| }) | |
| else: | |
| # Check 3: Translation text is not empty | |
| text = ayah['translations'][trans_key].get('text', '').strip() | |
| if not text: | |
| has_issues = True | |
| issue = f"Ayah {ayah_no} has empty {trans_key}" | |
| if issue not in surah_issues: | |
| surah_issues.append(issue) | |
| empty_text_issues.append({ | |
| 'surah_no': surah_no, | |
| 'surah_name': surah_name, | |
| 'ayah_no': ayah_no, | |
| 'translation': trans_key | |
| }) | |
| # Print result | |
| if has_issues: | |
| incomplete_surahs.append(surah_no) | |
| print(f"⚠️ Surah {surah_no:3d} ({surah_name:20s}): {', '.join(surah_issues[:3])}") | |
| if len(surah_issues) > 3: | |
| print(f" ... and {len(surah_issues) - 3} more issues") | |
| else: | |
| complete_surahs += 1 | |
| print(f"✅ Surah {surah_no:3d} ({surah_name:20s}): Complete ({actual_ayahs} ayahs, {len(EXPECTED_TRANSLATIONS)} translations)") | |
| # Final Summary | |
| print("\n" + "=" * 80) | |
| print("📊 VERIFICATION SUMMARY") | |
| print("=" * 80) | |
| print(f"✅ Complete Surahs: {complete_surahs}/{total_surahs} ({complete_surahs*100/total_surahs:.1f}%)") | |
| print(f"⚠️ Incomplete Surahs: {len(incomplete_surahs)}/{total_surahs}") | |
| print(f"❌ Missing Surahs: {len(missing_surahs)}/{total_surahs}") | |
| print(f"💥 Corrupted Files: {len(corrupted_files)}") | |
| print("\n" + "-" * 80) | |
| print("ISSUE BREAKDOWN:") | |
| print(f" Ayah count mismatches: {len(ayah_count_issues)}") | |
| print(f" Missing translations: {len(missing_translation_issues)}") | |
| print(f" Empty translation texts: {len(empty_text_issues)}") | |
| if missing_surahs: | |
| print(f"\n❌ MISSING SURAHS: {', '.join(map(str, missing_surahs))}") | |
| if corrupted_files: | |
| print(f"\n💥 CORRUPTED FILES:") | |
| for cf in corrupted_files: | |
| print(f" - {cf['file']}: {cf['error']}") | |
| if incomplete_surahs: | |
| print(f"\n⚠️ INCOMPLETE SURAHS ({len(incomplete_surahs)}):") | |
| for surah_no in incomplete_surahs[:20]: # Show first 20 | |
| print(f" - Surah {surah_no}") | |
| if len(incomplete_surahs) > 20: | |
| print(f" ... and {len(incomplete_surahs) - 20} more") | |
| # Detailed issue reports (if any) | |
| if ayah_count_issues: | |
| print(f"\n📊 AYAH COUNT ISSUES ({len(ayah_count_issues)}):") | |
| for issue in ayah_count_issues[:10]: | |
| print(f" - Surah {issue['surah_no']} ({issue['name']}): Expected {issue['expected']}, got {issue['actual']}") | |
| if len(ayah_count_issues) > 10: | |
| print(f" ... and {len(ayah_count_issues) - 10} more") | |
| if missing_translation_issues: | |
| print(f"\n🔤 MISSING TRANSLATION ISSUES ({len(missing_translation_issues)}):") | |
| # Group by translation key | |
| by_trans = {} | |
| for issue in missing_translation_issues: | |
| key = issue['missing'] | |
| by_trans.setdefault(key, []).append(issue) | |
| for trans_key, issues in by_trans.items(): | |
| print(f" {trans_key}: {len(issues)} ayahs missing") | |
| for issue in issues[:5]: | |
| print(f" - Surah {issue['surah_no']} ({issue['surah_name']}), Ayah {issue['ayah_no']}") | |
| if len(issues) > 5: | |
| print(f" ... and {len(issues) - 5} more") | |
| if empty_text_issues: | |
| print(f"\n📝 EMPTY TEXT ISSUES ({len(empty_text_issues)}):") | |
| for issue in empty_text_issues[:10]: | |
| print(f" - Surah {issue['surah_no']} ({issue['surah_name']}), Ayah {issue['ayah_no']}: {issue['translation']}") | |
| if len(empty_text_issues) > 10: | |
| print(f" ... and {len(empty_text_issues) - 10} more") | |
| print("\n" + "=" * 80) | |
| # Final verdict | |
| if complete_surahs == total_surahs: | |
| print("🎉 ALL FILES ARE COMPLETE AND VALID!") | |
| elif complete_surahs >= 110: | |
| print("✅ MOSTLY COMPLETE - Only minor issues") | |
| elif complete_surahs >= 100: | |
| print("⚠️ SOME ISSUES FOUND - Review recommended") | |
| else: | |
| print("❌ SIGNIFICANT ISSUES FOUND - Re-download recommended") | |
| print("=" * 80) | |
| return { | |
| 'complete': complete_surahs, | |
| 'incomplete': len(incomplete_surahs), | |
| 'missing': len(missing_surahs), | |
| 'corrupted': len(corrupted_files), | |
| 'total_issues': len(ayah_count_issues) + len(missing_translation_issues) + len(empty_text_issues) | |
| } | |
| if __name__ == "__main__": | |
| verify_all_files() | |