ocr-tech-k2 / languages /nepal-v1.json
prosperious's picture
Add 51 files (.gitignore ...)
1c43956 verified
Raw History Blame Contribute Delete
21.6 kB
{
"registry_id": "nepal-v1",
"description": "Nepal multilingual OCR target registry. Limbu/Sirijonga is the first non-Devanagari target; English is included for preservation but does not count toward the Nepal-language target.",
"minimum_nepal_language_count": 10,
"created_for": "PaddleOCR script-aware fine-tuning, synthesis, and real-document validation",
"claim_policy": "Synthetic data is allowed for training and ablations only. SOTA claims require held-out real/public validation artifacts.",
"languages": [
{
"id": "limbu",
"name": "Limbu / Yakthung",
"endonym": "ᤕᤠᤰᤌᤢᤱ",
"iso_639_3": "lif",
"priority": 1,
"counts_toward_nepal_language_target": true,
"first_target": true,
"scripts": [
{
"script": "Limbu / Sirijonga",
"primary": true,
"status": "encoded",
"unicode_ranges": ["U+1900..U+194F"],
"notes": "First non-Devanagari OCR target. Requires renderability and dictionary audits before training admission."
},
{
"script": "Devanagari",
"primary": false,
"status": "fallback_transcription",
"unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"],
"notes": "Useful for parallel/transcribed resources, not the primary Limbu-script claim target."
}
],
"font_candidates": [
{"name": "Noto Sans Limbu", "path": "/System/Library/Fonts/Supplemental/NotoSansLimbu-Regular.ttf", "status": "pending_render_audit"},
{"name": "local-ltk-fonts", "path": "/Users/cdjk/github/llm/ltk/fonts.zip", "status": "legacy_archive_pending_coverage_audit"}
],
"paddle_dictionary": "data/dictionaries/limbu-sirijonga-v1.txt",
"normalization": {"unicode_form": "NFC", "policy": "preserve_limbu_letters_and_signs"},
"corpora": [
{"name": "LTK Limbu resources", "path": "/Users/cdjk/github/llm/ltk/limbu", "host": "cdjk", "type": "local_verified_directory", "status": "schema_audit_pending"},
{"name": "Limdic dictionary", "path": "/Users/cdjk/github/llm/ltk/limbu/data/limdic_dictionary.json", "host": "cdjk", "type": "lexicon", "status": "exists_local"},
{"name": "Pangloss dictionary TSV", "path": "/Users/cdjk/github/llm/ltk/limbu/data/pangloss_dictionary.tsv", "host": "cdjk", "type": "lexicon", "status": "exists_local"}
],
"validation_sources": [
{"name": "Gorkhapatra/Naya Nepal article images", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "pending_manual_label"},
{"name": "Gorkhapatra epaper PDFs", "path": "data/gorkhapatra-pack-smoke", "type": "real_document_validation_pack", "status": "pending_manual_label"}
],
"gorkhapatra_language_labels": ["लिम्बु"],
"notes": "Primary research target for low-resource script-aware PaddleOCR adaptation."
},
{
"id": "nepali",
"name": "Nepali",
"endonym": "नेपाली",
"iso_639_3": "nep",
"priority": 2,
"counts_toward_nepal_language_target": true,
"first_target": false,
"scripts": [
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Established base script and regression-protection target."}
],
"font_candidates": [
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
],
"paddle_dictionary": "data/dictionaries/nepali-devanagari-v1.txt",
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
"corpora": [
{"name": "Nepali hunspell dictionary", "path": "/Users/cdjk/github/llm/ltk/ne_NP.dic", "host": "cdjk", "type": "lexicon", "status": "exists_local"}
],
"validation_sources": [
{"name": "Gorkhapatra epaper PDFs", "path": "data/gorkhapatra-pack-smoke", "type": "real_document_validation_pack", "status": "pending_manual_label"}
],
"gorkhapatra_language_labels": ["नेपाली"],
"notes": "Must not regress versus stock PaddleOCR/Tesseract/Surya baselines."
},
{
"id": "tamang",
"name": "Tamang",
"endonym": "तामाङ",
"iso_639_3": "taj",
"priority": 3,
"counts_toward_nepal_language_target": true,
"first_target": false,
"scripts": [
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari resources; dialect coverage must be tracked separately."}
],
"font_candidates": [
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
],
"paddle_dictionary": "data/dictionaries/tamang-devanagari-v1.txt",
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
"corpora": [
{"name": "LTK Tamang resources", "path": "/Users/cdjk/github/llm/ltk/tamang", "host": "cdjk", "type": "local_verified_directory", "status": "schema_audit_pending"},
{"name": "Western Tamang dictionary TSV", "path": "/Users/cdjk/github/llm/ltk/tamang/data/western_tamang_dictionary.tsv", "host": "cdjk", "type": "lexicon", "status": "exists_local"},
{"name": "NepTam80K train/test CSV", "path": "/Users/cdjk/github/llm/ltk/tamang/data", "host": "cdjk", "type": "parallel_or_synthetic_text", "status": "schema_audit_pending"}
],
"validation_sources": [
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_audit_pending"}
],
"gorkhapatra_language_labels": ["तामाङ"],
"notes": "High-priority Devanagari low-resource language with existing local resources."
},
{
"id": "magar",
"name": "Magar",
"endonym": "मगर",
"iso_639_3": "mrd",
"priority": 4,
"counts_toward_nepal_language_target": true,
"first_target": false,
"scripts": [
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
],
"font_candidates": [
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
],
"paddle_dictionary": "data/dictionaries/magar-devanagari-v1.txt",
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
"corpora": [
{"name": "LTK Magar resources", "path": "/Users/cdjk/github/llm/ltk/magar", "host": "cdjk", "type": "local_verified_directory", "status": "schema_audit_pending"},
{"name": "Western Magar dictionary TSV", "path": "/Users/cdjk/github/llm/ltk/magar/data/western_magar_dictionary.tsv", "host": "cdjk", "type": "lexicon", "status": "exists_local"}
],
"validation_sources": [
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
],
"gorkhapatra_language_labels": ["मगर"],
"notes": "Seen in Naya Nepal audit labels; needs manual reference pack."
},
{
"id": "wambule",
"name": "Wambule",
"endonym": "वाम्बुले",
"iso_639_3": "wme",
"priority": 5,
"counts_toward_nepal_language_target": true,
"first_target": false,
"scripts": [
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
],
"font_candidates": [
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
],
"paddle_dictionary": "data/dictionaries/wambule-devanagari-v1.txt",
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
"corpora": [
{"name": "Gorkhapatra/Naya Nepal Wambule articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
],
"validation_sources": [
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
],
"gorkhapatra_language_labels": ["वाम्बुले"],
"notes": "Real-source first; no local training corpus audited yet."
},
{
"id": "rana-tharu",
"name": "Rana Tharu",
"endonym": "राना थारु",
"iso_639_3": "thr",
"priority": 6,
"counts_toward_nepal_language_target": true,
"first_target": false,
"scripts": [
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
],
"font_candidates": [
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
],
"paddle_dictionary": "data/dictionaries/rana-tharu-devanagari-v1.txt",
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
"corpora": [
{"name": "Gorkhapatra/Naya Nepal Rana Tharu articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
],
"validation_sources": [
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
],
"gorkhapatra_language_labels": ["राना थारु", "थारु"],
"notes": "Track Rana Tharu separately from broader Tharu labels when references are created."
},
{
"id": "bantawa",
"name": "Bantawa / Rai",
"endonym": "बान्तावा",
"iso_639_3": "bap",
"priority": 7,
"counts_toward_nepal_language_target": true,
"first_target": false,
"scripts": [
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."},
{"script": "Kirat Rai", "primary": false, "status": "requires_unicode_font_audit", "unicode_ranges": ["U+16D40..U+16D7F"], "notes": "Potential future script target; do not train or claim until font/runtime support is verified."}
],
"font_candidates": [
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
],
"paddle_dictionary": "data/dictionaries/bantawa-devanagari-v1.txt",
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
"corpora": [
{"name": "Gorkhapatra/Naya Nepal Bantawa articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
],
"validation_sources": [
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
],
"gorkhapatra_language_labels": ["बान्तावा"],
"notes": "Rai-language coverage must stay language-specific; do not collapse all Rai languages into one metric."
},
{
"id": "kulung",
"name": "Kulung",
"endonym": "कुलुङ",
"iso_639_3": "kle",
"priority": 8,
"counts_toward_nepal_language_target": true,
"first_target": false,
"scripts": [
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
],
"font_candidates": [
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
],
"paddle_dictionary": "data/dictionaries/kulung-devanagari-v1.txt",
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
"corpora": [
{"name": "Gorkhapatra/Naya Nepal Kulung articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
],
"validation_sources": [
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
],
"gorkhapatra_language_labels": ["कुलुङ"],
"notes": "Real-source first; needs corpus expansion."
},
{
"id": "yakkha",
"name": "Yakkha",
"endonym": "याक्खा",
"iso_639_3": "ybh",
"priority": 9,
"counts_toward_nepal_language_target": true,
"first_target": false,
"scripts": [
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
],
"font_candidates": [
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
],
"paddle_dictionary": "data/dictionaries/yakkha-devanagari-v1.txt",
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
"corpora": [
{"name": "Gorkhapatra/Naya Nepal Yakkha articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
],
"validation_sources": [
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
],
"gorkhapatra_language_labels": ["याक्खा"],
"notes": "Real-source first; needs corpus expansion."
},
{
"id": "bhojpuri",
"name": "Bhojpuri",
"endonym": "भोजपुरी",
"iso_639_3": "bho",
"priority": 10,
"counts_toward_nepal_language_target": true,
"first_target": false,
"scripts": [
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
],
"font_candidates": [
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
],
"paddle_dictionary": "data/dictionaries/bhojpuri-devanagari-v1.txt",
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
"corpora": [
{"name": "Gorkhapatra/Naya Nepal Bhojpuri articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
],
"validation_sources": [
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
],
"gorkhapatra_language_labels": ["भोजपुरी"],
"notes": "Large-language baseline for Devanagari multilingual transfer."
},
{
"id": "santhali",
"name": "Santhali",
"endonym": "सन्थाली",
"iso_639_3": "sat",
"priority": 11,
"counts_toward_nepal_language_target": true,
"first_target": false,
"scripts": [
{"script": "Ol Chiki", "primary": true, "status": "encoded", "unicode_ranges": ["U+1C50..U+1C7F"], "notes": "Own-script target after Limbu; needs font/render audit."},
{"script": "Devanagari", "primary": false, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "May appear in Devanagari-transcribed sources."}
],
"font_candidates": [
{"name": "Noto Sans Ol Chiki", "path": "data/fonts/NotoSansOlChiki-Regular.ttf", "status": "pending_render_audit"},
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
],
"paddle_dictionary": "data/dictionaries/santhali-ol-chiki-v1.txt",
"normalization": {"unicode_form": "NFC", "policy": "preserve_ol_chiki_letters"},
"corpora": [
{"name": "Gorkhapatra/Naya Nepal Santhali articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
],
"validation_sources": [
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
],
"gorkhapatra_language_labels": ["सन्थाली"],
"notes": "Second own-script candidate after Limbu if real rendered sources and fonts are available."
},
{
"id": "maithili",
"name": "Maithili",
"endonym": "मैथिली",
"iso_639_3": "mai",
"priority": 12,
"counts_toward_nepal_language_target": true,
"first_target": false,
"scripts": [
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."},
{"script": "Tirhuta", "primary": false, "status": "future_audit", "unicode_ranges": ["U+11480..U+114DF"], "notes": "Own-script support is a future target and not part of the v1 claim without real validation."}
],
"font_candidates": [
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
],
"paddle_dictionary": "data/dictionaries/maithili-devanagari-v1.txt",
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
"corpora": [
{"name": "Maithili corpus discovery", "path": "TBD", "type": "external_corpus_pending", "status": "not_audited"}
],
"validation_sources": [
{"name": "Gorkhapatra/Naya Nepal or public documents", "path": "TBD", "type": "real_document_discovery", "status": "not_audited"}
],
"gorkhapatra_language_labels": ["मैथिली"],
"notes": "Included because it is a major Nepal language; needs source discovery before training."
},
{
"id": "english",
"name": "English",
"endonym": "English",
"iso_639_3": "eng",
"priority": 99,
"counts_toward_nepal_language_target": false,
"first_target": false,
"scripts": [
{"script": "Latin", "primary": true, "status": "encoded", "unicode_ranges": ["U+0000..U+007F", "U+0080..U+00FF"], "notes": "Regression-protection target only."}
],
"font_candidates": [
{"name": "Arial Unicode MS", "path": "/Library/Fonts/Arial Unicode.ttf", "status": "pending_render_audit"},
{"name": "Helvetica", "path": "/System/Library/Fonts/Helvetica.ttc", "status": "pending_render_audit"}
],
"paddle_dictionary": "data/dictionaries/english-latin-v1.txt",
"normalization": {"unicode_form": "NFC", "policy": "preserve_ascii_and_common_latin"},
"corpora": [
{"name": "NVIDIA OCR Synthetic Multilingual English subset", "path": "huggingface:nvidia/OCR-Synthetic-Multilingual-v1", "type": "synthetic_line_crops", "status": "schema_audit_required_before_use"}
],
"validation_sources": [
{"name": "Stock English OCR eval subset", "path": "TBD", "type": "regression_eval", "status": "not_audited"}
],
"gorkhapatra_language_labels": [],
"notes": "Included to protect English accuracy in mixed documents; not counted as a Nepal-language target."
}
]
}