Upload all models and assets for af (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: af
-language_name: AF
+language_name: Afrikaans
 language_family: germanic_west_anglofrisian
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-germanic_west_anglofrisian
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -26,17 +36,17 @@ metrics:
     value: 4.620
   - name: best_isotropy
     type: isotropy
-    value: 0.6959
+    value: 0.6974
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# AF - Wikilangs Models
+# Afrikaans - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **AF** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Afrikaans** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,41 +90,41 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.747x | 3.75 | 0.0650% | 1,240,279 |
-| **16k** | 4.108x | 4.11 | 0.0712% | 1,131,351 |
-| **32k** | 4.402x | 4.40 | 0.0763% | 1,055,895 |
-| **64k** | 4.620x 🏆 | 4.62 | 0.0801% | 1,006,125 |
+| **8k** | 3.748x | 3.75 | 0.0650% | 1,240,703 |
+| **16k** | 4.108x | 4.11 | 0.0712% | 1,132,029 |
+| **32k** | 4.402x | 4.40 | 0.0763% | 1,056,512 |
+| **64k** | 4.620x 🏆 | 4.62 | 0.0801% | 1,006,543 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Neede is ’n dorp in die munisipaliteit Berkelland in die provinsie Gelderland in...`
+**Sample 1:** `Electron is 'n industriële gebied in Johannesburg, Suid-Afrika. Verwysings van J...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁ne e de ▁is ▁’ n ▁dorp ▁in ▁die ▁munisipaliteit ... (+14 more)` | 24 |
-| 16k | `▁ne ede ▁is ▁’ n ▁dorp ▁in ▁die ▁munisipaliteit ▁berk ... (+13 more)` | 23 |
-| 32k | `▁ne ede ▁is ▁’ n ▁dorp ▁in ▁die ▁munisipaliteit ▁berk ... (+13 more)` | 23 |
-| 64k | `▁ne ede ▁is ▁’ n ▁dorp ▁in ▁die ▁munisipaliteit ▁berkelland ... (+12 more)` | 22 |
+| 8k | `▁electr on ▁is ▁' n ▁industr iële ▁gebied ▁in ▁johannesburg ... (+8 more)` | 18 |
+| 16k | `▁electr on ▁is ▁' n ▁industriële ▁gebied ▁in ▁johannesburg , ... (+7 more)` | 17 |
+| 32k | `▁electr on ▁is ▁' n ▁industriële ▁gebied ▁in ▁johannesburg , ... (+7 more)` | 17 |
+| 64k | `▁electron ▁is ▁' n ▁industriële ▁gebied ▁in ▁johannesburg , ▁suid ... (+6 more)` | 16 |
 
-**Sample 2:** `Japan Nasionale Roete 210 is 'n nasionale snelweg in Japan. Verwysings paaie in ...`
+**Sample 2:** `Fig Tree Creek is 'n takrivier van die Kaaprivier in Mpumalanga in Suid-Afrika. ...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁japan ▁nasionale ▁roete ▁ 2 1 0 ▁is ▁' n ... (+9 more)` | 19 |
-| 16k | `▁japan ▁nasionale ▁roete ▁ 2 1 0 ▁is ▁' n ... (+9 more)` | 19 |
-| 32k | `▁japan ▁nasionale ▁roete ▁ 2 1 0 ▁is ▁' n ... (+9 more)` | 19 |
-| 64k | `▁japan ▁nasionale ▁roete ▁ 2 1 0 ▁is ▁' n ... (+9 more)` | 19 |
+| 8k | `▁fig ▁tree ▁c reek ▁is ▁' n ▁tak rivier ▁van ... (+22 more)` | 32 |
+| 16k | `▁fig ▁tree ▁creek ▁is ▁' n ▁tak rivier ▁van ▁die ... (+20 more)` | 30 |
+| 32k | `▁fig ▁tree ▁creek ▁is ▁' n ▁takrivier ▁van ▁die ▁kaap ... (+19 more)` | 29 |
+| 64k | `▁fig ▁tree ▁creek ▁is ▁' n ▁takrivier ▁van ▁die ▁kaap ... (+19 more)` | 29 |
 
-**Sample 3:** `Ja'Net DuBois (gebore 5 Augustus – 17 Februarie was 'n Amerikaanse aktrise. Ekst...`
+**Sample 3:** `Japan Nasionale Roete 390 is 'n nasionale snelweg in Japan. Verwysings paaie in ...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁ja ' net ▁dub ois ▁( gebore ▁ 5 ▁augustus ... (+30 more)` | 40 |
-| 16k | `▁ja ' net ▁dub ois ▁( gebore ▁ 5 ▁augustus ... (+30 more)` | 40 |
-| 32k | `▁ja ' net ▁dub ois ▁( gebore ▁ 5 ▁augustus ... (+30 more)` | 40 |
-| 64k | `▁ja ' net ▁dubois ▁( gebore ▁ 5 ▁augustus ▁– ... (+29 more)` | 39 |
+| 8k | `▁japan ▁nasionale ▁roete ▁ 3 9 0 ▁is ▁' n ... (+9 more)` | 19 |
+| 16k | `▁japan ▁nasionale ▁roete ▁ 3 9 0 ▁is ▁' n ... (+9 more)` | 19 |
+| 32k | `▁japan ▁nasionale ▁roete ▁ 3 9 0 ▁is ▁' n ... (+9 more)` | 19 |
+| 64k | `▁japan ▁nasionale ▁roete ▁ 3 9 0 ▁is ▁' n ... (+9 more)` | 19 |
 
 
 ### Key Findings
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 67,018 | 16.03 | 738,183 | 13.7% | 29.1% |
-| **2-gram** | Subword | 253 🏆 | 7.98 | 13,576 | 69.5% | 99.3% |
-| **3-gram** | Word | 293,932 | 18.17 | 1,499,483 | 5.8% | 16.9% |
-| **3-gram** | Subword | 2,161 | 11.08 | 96,263 | 28.5% | 71.9% |
-| **4-gram** | Word | 555,388 | 19.08 | 2,510,434 | 6.5% | 16.6% |
-| **4-gram** | Subword | 12,658 | 13.63 | 531,540 | 15.0% | 40.0% |
+| **2-gram** | Word | 67,167 | 16.04 | 741,646 | 13.7% | 29.1% |
+| **2-gram** | Subword | 253 🏆 | 7.98 | 13,611 | 69.5% | 99.3% |
+| **3-gram** | Word | 295,297 | 18.17 | 1,507,746 | 5.8% | 16.9% |
+| **3-gram** | Subword | 2,160 | 11.08 | 96,463 | 28.5% | 71.9% |
+| **4-gram** | Word | 559,011 | 19.09 | 2,524,344 | 6.5% | 16.5% |
+| **4-gram** | Subword | 12,656 | 13.63 | 532,733 | 15.0% | 40.0% |
+| **5-gram** | Word | 326,109 | 18.31 | 1,744,378 | 9.4% | 21.4% |
+| **5-gram** | Subword | 52,200 | 15.67 | 1,835,021 | 9.1% | 25.1% |
 
 ### Top 5 N-grams by Size
 
@@ -150,68 +162,88 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `van die` | 509,583 |
-| 2 | `in die` | 342,810 |
-| 3 | `is n` | 114,159 |
-| 4 | `en die` | 109,201 |
-| 5 | `is die` | 91,083 |
+| 1 | `van die` | 511,917 |
+| 2 | `in die` | 344,470 |
+| 3 | `is n` | 115,009 |
+| 4 | `en die` | 109,902 |
+| 5 | `is die` | 91,555 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `van suid afrika` | 26,860 |
-| 2 | `rolle in die` | 25,216 |
-| 3 | `die 20ste eeu` | 24,460 |
-| 4 | `van die 20ste` | 23,487 |
-| 5 | `eksterne skakels in` | 22,326 |
+| 1 | `van suid afrika` | 27,044 |
+| 2 | `rolle in die` | 25,215 |
+| 3 | `die 20ste eeu` | 24,473 |
+| 4 | `van die 20ste` | 23,498 |
+| 5 | `eksterne skakels in` | 22,336 |
 
 **4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `van die 20ste eeu` | 23,423 |
-| 2 | `manlike akteurs van die` | 20,397 |
+| 1 | `van die 20ste eeu` | 23,435 |
+| 2 | `manlike akteurs van die` | 20,400 |
 | 3 | `rolle in die rolprente` | 19,639 |
-| 4 | `van die 21ste eeu` | 15,799 |
-| 5 | `plants of the world` | 13,996 |
+| 4 | `van die 21ste eeu` | 15,805 |
+| 5 | `plants of the world` | 14,447 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `bekend vir sy rolle in` | 13,780 |
+| 2 | `vir sy rolle in die` | 13,771 |
+| 3 | `akteurs van die 20ste eeu` | 12,560 |
+| 4 | `manlike akteurs van die 20ste` | 12,536 |
+| 5 | `plants of the world online` | 11,731 |
 
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `e _` | 8,883,972 |
-| 2 | `n _` | 5,845,355 |
-| 3 | `i e` | 5,296,532 |
-| 4 | `e r` | 4,795,609 |
-| 5 | `_ d` | 4,496,380 |
+| 1 | `e _` | 8,931,762 |
+| 2 | `n _` | 5,874,572 |
+| 3 | `i e` | 5,325,847 |
+| 4 | `e r` | 4,823,982 |
+| 5 | `_ d` | 4,520,196 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `i e _` | 3,582,000 |
-| 2 | `_ d i` | 3,169,450 |
-| 3 | `d i e` | 3,046,581 |
-| 4 | `a n _` | 1,886,278 |
-| 5 | `e n _` | 1,538,281 |
+| 1 | `i e _` | 3,601,485 |
+| 2 | `_ d i` | 3,186,521 |
+| 3 | `d i e` | 3,062,960 |
+| 4 | `a n _` | 1,896,257 |
+| 5 | `e n _` | 1,548,169 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `d i e _` | 2,916,346 |
-| 2 | `_ d i e` | 2,836,188 |
-| 3 | `_ v a n` | 1,357,382 |
-| 4 | `v a n _` | 1,341,795 |
-| 5 | `n _ d i` | 1,169,352 |
+| 1 | `d i e _` | 2,931,996 |
+| 2 | `_ d i e` | 2,851,512 |
+| 3 | `_ v a n` | 1,364,018 |
+| 4 | `v a n _` | 1,348,393 |
+| 5 | `n _ d i` | 1,174,871 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `_ d i e _` | 2,794,095 |
+| 2 | `_ v a n _` | 1,320,773 |
+| 3 | `n _ d i e` | 1,131,268 |
+| 4 | `a n _ d i` | 628,822 |
+| 5 | `v a n _ d` | 564,996 |
 
 
 ### Key Findings
 
 - **Best Perplexity:** 2-gram (subword) with 253
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~40% of corpus
+- **Coverage:** Top-1000 patterns cover ~25% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.9426 | 1.922 | 9.97 | 884,548 | 5.7% |
-| **1** | Subword | 1.0721 | 2.102 | 6.58 | 7,654 | 0.0% |
-| **2** | Word | 0.3842 | 1.305 | 2.33 | 8,810,967 | 61.6% |
-| **2** | Subword | 0.7311 | 1.660 | 4.61 | 50,359 | 26.9% |
-| **3** | Word | 0.1707 | 1.126 | 1.40 | 20,525,798 | 82.9% |
-| **3** | Subword | 0.7061 | 1.631 | 4.02 | 231,918 | 29.4% |
-| **4** | Word | 0.0704 🏆 | 1.050 | 1.13 | 28,628,609 | 93.0% |
-| **4** | Subword | 0.6911 | 1.615 | 3.50 | 931,942 | 30.9% |
+| **1** | Word | 0.9424 | 1.922 | 9.98 | 888,057 | 5.8% |
+| **1** | Subword | 1.0749 | 2.107 | 6.60 | 7,659 | 0.0% |
+| **2** | Word | 0.3845 | 1.305 | 2.33 | 8,849,236 | 61.6% |
+| **2** | Subword | 0.7312 | 1.660 | 4.61 | 50,492 | 26.9% |
+| **3** | Word | 0.1708 | 1.126 | 1.40 | 20,626,048 | 82.9% |
+| **3** | Subword | 0.7057 | 1.631 | 4.02 | 232,520 | 29.4% |
+| **4** | Word | 0.0705 🏆 | 1.050 | 1.13 | 28,778,158 | 92.9% |
+| **4** | Subword | 0.6912 | 1.615 | 3.50 | 934,149 | 30.9% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `die burger 13 augustus se fsa nommer vier soldate die uitkoms vir letterkunde in n klein`
-2. `van soest r amphoriscus cylindrus is in paradise careful he du mont dolent teen 5 6`
-3. `in die rigting van die twee broers en met die ou teeroete die liberte het die`
+1. `die dr g mineur d ilse ná dié samewerking met 46 155 173 minute met ywer`
+2. `van president trump het hierdie maniak nie voortsetting van die verbranding maak in die spesie is`
+3. `in te veel van kaiserstuhl gebied rondom die farao self deur die nasionalistiese en geofiet wat`
 
 **Context Size 2:**
 
-1. `van die spons behoort tot die genus geodia en tot die genus leucadendron behoort en is deur`
-2. `in die stille oseaan wat tot 4 uur later onder westerse intellektuele invloede gekom frankryk hertog...`
-3. `is n nuwe telling van 83 etse wat spesiaal vir hierdie liedjie is in die rolprente innerspace`
+1. `van die eufraat te gaan om die lewe geroep om n nuwe uitgawe cambridge university press princeton`
+2. `in die swartberge en die patrone diagonaal 2 4 brown bl 101 in suidoos asië panthera p`
+3. `is n blouwit ster dit is egter vas gekant teen die middel van toenemende afvalligheid te volhard`
 
 **Context Size 3:**
 
-1. `rolle in die rolprente the squaw man resurrección kongo the broken wing roaring rails en devils dice...`
-2. `van die 20ste eeu aktrises van die 21ste eeu mense aktrises van die 21ste eeu aktrises van die`
-3. `eksterne skakels in manlike akteurs van die 20ste eeu in n stormwind deur pieter kluyver wind is die`
+1. `rolle in die rolprente kitty foyle missile to the moon tour aangekondig n amptelike konserttoer met ...`
+2. `van die 20ste eeu manlike akteurs van die 21ste eeu aktrises van die 21ste eeu manlike akteurs van`
+3. `eksterne skakels in in manlike akteurs van die 20ste eeu manlike akteurs van die 20ste eeu aktrises ...`
 
 **Context Size 4:**
 
-1. `manlike akteurs van die 20ste eeu aktrises van die 20ste eeu rolprentvervaardigers in mense van die ...`
-2. `rolle in die rolprente tomorrow when the war began the weekend shift high life tidelands eksterne sk...`
-3. `plants of the world online van suid afrika plante van suid afrika gramineum`
+1. `manlike akteurs van die 21ste eeu manlike akteurs van die 20ste eeu byna uitgeroei is die oorspronkl...`
+2. `rolle in die rolprente batman the movie scream evelyn scream televisiereekse playhouse 90 frontier d...`
+3. `plants of the world online van namibië van suid afrika van die tweede vryheidsoorlog die eerste is b...`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `_ho_evise_j._wom`
-2. `enstein_n_nkt_he`
-3. `igeked_dig_linid`
+1. `_&_ligesagetiebe`
+2. `e_n_dnore_drs_va`
+3. `ie_wogerct_wache`
 
 **Context Size 2:**
 
-1. `e_nin:_sy_offie_l`
-2. `n_die_nivir_nbom_`
-3. `ie_uikaide_ver_ro`
+1. `e_van_gesede_wasc`
+2. `n_baiensomenaar,_`
+3. `ierk_ing_maaktors`
 
 **Context Size 3:**
 
-1. `ie_van_die_bespelt`
-2. `_die_wassen_paropo`
-3. `die_van_spel_andar`
+1. `ie_te_sies_die_in_`
+2. `_die_redig_gebruit`
+3. `die_alber_ds._hy_w`
 
 **Context Size 4:**
 
-1. `die_vonnikeksadige_`
-2. `_die_branse_levisie`
-3. `_van_waar_toest,_r.`
+1. `die_rolle_wêreld_en`
+2. `_die_se_limitiek_di`
+3. `_van_'n_albei_dat_h`
 
 
 ### Key Findings
 
-- **Best Predictability:** Context-4 (word) with 93.0% predictability
+- **Best Predictability:** Context-4 (word) with 92.9% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (931,942 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (934,149 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,48 +346,48 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 403,515 |
-| Total Tokens | 38,429,571 |
-| Mean Frequency | 95.24 |
+| Vocabulary Size | 404,957 |
+| Total Tokens | 38,641,442 |
+| Mean Frequency | 95.42 |
 | Median Frequency | 4 |
-| Frequency Std Dev | 6117.62 |
+| Frequency Std Dev | 6141.00 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | die | 2,828,931 |
-| 2 | van | 1,318,980 |
-| 3 | in | 1,109,973 |
-| 4 | en | 1,045,922 |
-| 5 | n | 802,080 |
-| 6 | is | 763,111 |
-| 7 | het | 641,876 |
-| 8 | wat | 341,748 |
-| 9 | the | 292,778 |
-| 10 | op | 289,154 |
+| 1 | die | 2,844,119 |
+| 2 | van | 1,325,435 |
+| 3 | in | 1,115,990 |
+| 4 | en | 1,052,538 |
+| 5 | n | 806,584 |
+| 6 | is | 768,312 |
+| 7 | het | 648,164 |
+| 8 | wat | 343,988 |
+| 9 | the | 293,953 |
+| 10 | op | 290,589 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | williamsville | 2 |
-| 2 | 1kd | 2 |
-| 3 | argiefkopie | 2 |
-| 4 | liuzhi | 2 |
-| 5 | microsat | 2 |
-| 6 | orbex | 2 |
-| 7 | afrikanertoekoms | 2 |
-| 8 | wêreldkennis | 2 |
-| 9 | gastebydraes | 2 |
-| 10 | sandkweek | 2 |
+| 1 | bajnokság | 2 |
+| 2 | zalaegerszegi | 2 |
+| 3 | akteurskategorieë | 2 |
+| 4 | mullens | 2 |
+| 5 | grafiekstruktuur | 2 |
+| 6 | roostergrafieke | 2 |
+| 7 | sokkerbekertitels | 2 |
+| 8 | chalobah | 2 |
+| 9 | sentrumverdediger | 2 |
+| 10 | guðjohnsen | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
 | Zipf Coefficient | 1.0518 |
-| R² (Goodness of Fit) | 0.996010 |
+| R² (Goodness of Fit) | 0.995983 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
@@ -371,7 +403,7 @@ Below are text samples generated from each subword-based Markov chain model:
 
 - **Zipf Compliance:** R²=0.9960 indicates excellent adherence to Zipf's law
 - **High Frequency Dominance:** Top 100 words cover 43.7% of corpus
-- **Long Tail:** 393,515 words needed for remaining 15.0% coverage
+- **Long Tail:** 394,957 words needed for remaining 15.0% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.6926 | 0.3664 | N/A | N/A |
-| **mono_64d** | 64 | 0.6959 🏆 | 0.3037 | N/A | N/A |
-| **mono_128d** | 128 | 0.6723 | 0.2366 | N/A | N/A |
+| **mono_32d** | 32 | 0.6861 | 0.3709 | N/A | N/A |
+| **mono_64d** | 64 | 0.6974 | 0.2860 | N/A | N/A |
+| **mono_128d** | 128 | 0.6739 | 0.2351 | N/A | N/A |
+| **aligned_32d** | 32 | 0.6861 | 0.3805 | 0.3500 | 0.6860 |
+| **aligned_64d** | 64 | 0.6974 🏆 | 0.2901 | 0.5440 | 0.8400 |
+| **aligned_128d** | 128 | 0.6739 | 0.2381 | 0.6160 | 0.8900 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_64d with 0.6959 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.3023. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** aligned_64d with 0.6974 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.3001. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 61.6% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **-0.147** | Low formulaic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -426,19 +461,19 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Prefixes
 | Prefix | Examples |
 |--------|----------|
-| `-ge` | gewandel, gefloreer, getrouheidseed |
+| `-ma` | maanteorieë, markomgewing, mataiva |
 
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-e` | kurasse, kortikosteroïde, maatskappyname |
-| `-s` | tuttles, stakings, kenens |
-| `-er` | gefloreer, umbilorivier, koorsanger |
-| `-es` | tuttles, spectres, kladmetodes |
-| `-ng` | kruiskleding, saambring, swangerskapvergiftiging |
-| `-ie` | patagonie, photographie, kriminologie |
-| `-ing` | kruiskleding, saambring, swangerskapvergiftiging |
-| `-te` | monofisiete, skrikwekkendste, curriebekerpunte |
+| `-e` | squeeze, summerside, tirolse |
+| `-s` | repsyfers, sangkunstenaars, kananaskis |
+| `-er` | shaffer, ondier, skilpadkewer |
+| `-es` | langafstandroetes, treasuries, ferrities |
+| `-ng` | enkelstring, markomgewing, erlösung |
+| `-ing` | enkelstring, markomgewing, navorsingsbelangstelling |
+| `-te` | sudete, heroute, afleweringsdienste |
+| `-de` | summerside, geünieerde, uitgetrede |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -446,18 +481,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `pren` | 2.37x | 29 contexts | prent, prens, prend |
-| `staa` | 1.71x | 98 contexts | staat, staal, staan |
-| `ings` | 1.53x | 145 contexts | wings, rings, hings |
-| `brui` | 1.99x | 44 contexts | bruis, bruid, bruik |
-| `kend` | 1.65x | 95 contexts | kende, kendo, skend |
-| `ebru` | 2.08x | 32 contexts | gebru, hebrus, gebruk |
-| `ersk` | 1.54x | 107 contexts | perske, koersk, perski |
-| `erdi` | 1.61x | 84 contexts | verdi, ferdi, gerdi |
-| `rste` | 1.42x | 150 contexts | erste, eerste, fyrste |
-| `rdie` | 1.73x | 51 contexts | ardie, gordie, jordie |
-| `kste` | 1.54x | 71 contexts | ekster, tekste, dikste |
-| `eken` | 1.34x | 123 contexts | weken, deken, oeken |
+| `pren` | 2.42x | 29 contexts | prens, prent, prend |
+| `staa` | 1.70x | 98 contexts | staak, staas, staab |
+| `ings` | 1.49x | 146 contexts | lings, wings, hings |
+| `kend` | 1.58x | 95 contexts | kendo, kenda, kende |
+| `eken` | 1.48x | 124 contexts | teken, deken, reken |
+| `ebru` | 2.04x | 32 contexts | gebru, hebrus, cebrus |
+| `erdi` | 1.58x | 85 contexts | ferdi, serdi, verdi |
+| `brui` | 1.78x | 44 contexts | bruin, bruit, bruis |
+| `elik` | 1.53x | 82 contexts | melik, elika, lelik |
+| `aans` | 1.44x | 88 contexts | aansê, faans, maans |
+| `ersk` | 1.32x | 109 contexts | koersk, perski, perske |
+| `kste` | 1.42x | 71 contexts | ekster, dikste, rykste |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -465,16 +500,16 @@ This table shows which prefixes and suffixes most frequently co-occur on the sam
 
 | Prefix | Suffix | Frequency | Examples |
 |--------|--------|-----------|----------|
-| `-ge` | `-e` | 63 words | geenlokusse, gebruikskode |
-| `-ge` | `-de` | 28 words | gebruikskode, geeboniseerde |
-| `-ge` | `-er` | 27 words | geigenspieler, getelegrafeer |
-| `-ge` | `-s` | 11 words | gemeentesusters, geles |
-| `-ge` | `-en` | 9 words | gefahren, gelegen |
-| `-ge` | `-te` | 6 words | geskenkte, geweldigste |
-| `-ge` | `-ie` | 5 words | getalteorie, geelglasogie |
-| `-ge` | `-es` | 4 words | geles, geowetenskaplikes |
-| `-ge` | `-ng` | 2 words | geeking, gesondheidsbevordering |
-| `-ge` | `-ing` | 1 words | geeking, gesondheidsbevordering |
+| `-ma` | `-e` | 32 words | mapogsgrotte, malte |
+| `-ma` | `-s` | 24 words | magnesiumlegerings, maatskappybestuurders |
+| `-ma` | `-er` | 11 words | marineer, mansspeler |
+| `-ma` | `-ng` | 5 words | maksimalisering, magsdeling |
+| `-ma` | `-en` | 5 words | marten, maurren |
+| `-ma` | `-te` | 4 words | mapogsgrotte, malte |
+| `-ma` | `-se` | 4 words | majestueuse, manneristiese |
+| `-ma` | `-es` | 4 words | maccabees, maykersfees |
+| `-ma` | `-ing` | 3 words | maksimalisering, magsdeling |
+| `-ma` | `-de` | 2 words | malahide, mansonbendelede |
 
 ### 6.5 Recursive Morpheme Segmentation
 
@@ -482,26 +517,26 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
-| gemonteerde | **`ge-monte-er-de`** | 7.5 | `monte` |
-| bevredigende | **`bevredig-en-de`** | 6.0 | `bevredig` |
-| ouditering | **`oudit-er-ing`** | 6.0 | `oudit` |
-| kruiningen | **`kruin-ing-en`** | 6.0 | `kruin` |
-| verlorener | **`verlor-en-er`** | 6.0 | `verlor` |
-| verhardende | **`verhard-en-de`** | 6.0 | `verhard` |
-| bestuifde | **`bestuif-de`** | 4.5 | `bestuif` |
-| behoeften | **`behoeft-en`** | 4.5 | `behoeft` |
-| verminkte | **`vermink-te`** | 4.5 | `vermink` |
-| onreëlmatiger | **`onreëlmatig-er`** | 4.5 | `onreëlmatig` |
-| kollageen | **`kollage-en`** | 4.5 | `kollage` |
-| gekrummel | **`ge-krummel`** | 4.5 | `krummel` |
-| repeterende | **`repet-er-en-de`** | 4.5 | `repet` |
-| gehoorvermoë | **`ge-hoorvermoë`** | 4.5 | `hoorvermoë` |
-| eksoskelette | **`eksoskelet-te`** | 4.5 | `eksoskelet` |
+| durangense | **`dura-ng-en-se`** | 7.5 | `dura` |
+| bessinger | **`bess-ing-er`** | 6.0 | `bess` |
+| selflaaiende | **`selflaai-en-de`** | 6.0 | `selflaai` |
+| durlacher | **`durlach-er`** | 4.5 | `durlach` |
+| emotionen | **`emotion-en`** | 4.5 | `emotion` |
+| afgeperste | **`afgepers-te`** | 4.5 | `afgepers` |
+| apostelen | **`apostel-en`** | 4.5 | `apostel` |
+| kazachstanse | **`kazachstan-se`** | 4.5 | `kazachstan` |
+| afgerolde | **`afgerol-de`** | 4.5 | `afgerol` |
+| luggelanseerde | **`luggelan-se-er-de`** | 4.5 | `luggelan` |
+| verveling | **`vervel-ing`** | 4.5 | `vervel` |
+| biofiltrering | **`biofiltr-er-ing`** | 3.0 | `biofiltr` |
+| gefasiliteer | **`gefasili-te-er`** | 3.0 | `gefasili` |
+| palermosteen | **`palermos-te-en`** | 3.0 | `palermos` |
+| trekmense | **`trekm-en-se`** | 3.0 | `trekm` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language AF appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Afrikaans shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
 
 ---
 ## 7. Summary & Recommendations
@@ -514,7 +549,7 @@ The language AF appears to be more isolating or has a highly fixed vocabulary. W
 |-----------|-------------|-----------|
 | Tokenizer | **64k BPE** | Best compression (4.62x) |
 | N-gram | **2-gram** | Lowest perplexity (253) |
-| Markov | **Context-4** | Highest predictability (93.0%) |
+| Markov | **Context-4** | Highest predictability (92.9%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
 
@@ -728,4 +763,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 07:17:29*
+*Report Date: 2026-01-03 19:59:08*

models/embeddings/aligned/af_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e62d23c6e51b99389d05cb2158c8165748690ce7a7d005a13cc1157bbfad0d9
+size 1302456288

models/embeddings/aligned/af_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "af", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/af_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00f9d888dd324cf2cd64106642e8201c64cb75a72e23f81bba411c0c1d65cf35
+size 65664

models/embeddings/aligned/af_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "af",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 101796,
+  "vocab_size": 267090
+}

models/embeddings/aligned/af_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7cf6aad7f3c68c679646bdeb040f3bff12cb1c3c9bee206b94ce169e40198d8
+size 329331168

models/embeddings/aligned/af_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "af", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/af_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cf367b29395591e45bbac7199c321296ee76cc5e1b949cec8b69f8c33ff4bdb
+size 4224

models/embeddings/aligned/af_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "af",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 101796,
+  "vocab_size": 267090
+}

models/embeddings/aligned/af_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1441d615c03bfbfeecc113eef2e611095ed0a3a5ac2c0b588b5158c5310288dc
+size 653706208

models/embeddings/aligned/af_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "af", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/af_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4261509266d295aa2d72eced15227f9ba5f0a5cd8d995089442d6037f17e8d9
+size 16512

models/embeddings/aligned/af_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "af",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 101796,
+  "vocab_size": 267090
+}

models/embeddings/monolingual/af_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:11057ead6d145fd80aa8a3489e8c8d86000071008dba47a7f9f04188e658acb2
-size 1301432503
+oid sha256:4e62d23c6e51b99389d05cb2158c8165748690ce7a7d005a13cc1157bbfad0d9
+size 1302456288

models/embeddings/monolingual/af_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 266109
+  "vocab_size": 267090
 }

models/embeddings/monolingual/af_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:933902af3f06d4dccbab2f9cda13414a48c3a2fafa734a3e6c9cc651ffed6732
-size 329060791
+oid sha256:d7cf6aad7f3c68c679646bdeb040f3bff12cb1c3c9bee206b94ce169e40198d8
+size 329331168

models/embeddings/monolingual/af_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 266109
+  "vocab_size": 267090
 }

models/embeddings/monolingual/af_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c676e073cfbf1cb9b2e7f3e4b3e94463a17ee848c6d9df01ea4fc280e2f7ce6
-size 653184695
+oid sha256:1441d615c03bfbfeecc113eef2e611095ed0a3a5ac2c0b588b5158c5310288dc
+size 653706208

models/embeddings/monolingual/af_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 266109
+  "vocab_size": 267090
 }

models/subword_markov/af_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d149d8684419833d4d657f06ff43fde6f6330d2a78848a075e6c2fec57f7c4a8
-size 378608
+oid sha256:3819379bb72b6f342cd3be62dedeb81cda85065bb637a3430a290cd2ac146998
+size 375765

models/subword_markov/af_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "af",
-  "unique_contexts": 7654,
-  "total_transitions": 242297410
+  "unique_contexts": 7659,
+  "total_transitions": 243670715
 }

models/subword_markov/af_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:420f531b4f1b408ddda531f838f2d4932ecc9cbb7de16ed5589351bbace9b214
-size 1866665
+oid sha256:e7980810da30e31528b94d99c34186143ec57e1d31fca115d009898367d34316
+size 1851402

models/subword_markov/af_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "af",
-  "unique_contexts": 50359,
-  "total_transitions": 242170946
+  "unique_contexts": 50492,
+  "total_transitions": 243543429
 }

models/subword_markov/af_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:075a8bbd37d56e56e0b0f55526f7b478207748bf68daf7f1981287409f6c5cb8
-size 8218278
+oid sha256:f365aac7634bc74d0b11534dce48cb64c8df917f6703ba295776e5f846362f03
+size 8255986

models/subword_markov/af_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "af",
-  "unique_contexts": 231918,
-  "total_transitions": 242044482
+  "unique_contexts": 232520,
+  "total_transitions": 243416143
 }

models/subword_markov/af_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e71c7cbca9385f2aabc8bff2108e5cb3876745099e692d102b8817bd6e393051
-size 26401415
+oid sha256:3a04e015fa108bad17cca77b918084f1a6ea2335d89fc46a1364f6ace9a706b6
+size 26443804

models/subword_markov/af_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "af",
-  "unique_contexts": 931942,
-  "total_transitions": 241918018
+  "unique_contexts": 934149,
+  "total_transitions": 243288857
 }

models/subword_ngram/af_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69bbe527269e41324fd2369bdb8f0798f0b62fd22d70026c15fbf678f77788e1
-size 182664
+oid sha256:dad8d3d8adcf1b94b9bfed8cd88f882d51911e121fe5896c19a55e2eb5acc5c5
+size 183424

models/subword_ngram/af_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "af",
-  "unique_ngrams": 13576,
-  "total_ngrams": 242297410
+  "unique_ngrams": 13611,
+  "total_ngrams": 243670715
 }

models/subword_ngram/af_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8fcbc486b3961f9f14aee1492ea622129d621d205e682bab43457c4029f343ca
-size 1229801
+oid sha256:a654236a2fe8be07185f8b8ca8485460d88569b491b9195a2084711d28eea6b0
+size 1223369

models/subword_ngram/af_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "af",
-  "unique_ngrams": 96263,
-  "total_ngrams": 242170946
+  "unique_ngrams": 96463,
+  "total_ngrams": 243543429
 }

models/subword_ngram/af_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b35e67ae40780c953eae3c44b6344369d951fcbf2c26114557257a76b11f77d5
-size 6246285
+oid sha256:1daa4c098df46f5e060e468aeab63b397e1bc5caf477ca8f52df86ef8d226fb5
+size 6277243

models/subword_ngram/af_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "af",
-  "unique_ngrams": 531540,
-  "total_ngrams": 242044482
+  "unique_ngrams": 532733,
+  "total_ngrams": 243416143
 }

models/subword_ngram/af_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84cdbdb5194fed63ca4694c937a778de19fd293bae6e8c5ec6426e383f1cf7b3
+size 21994195

models/subword_ngram/af_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "af",
+  "unique_ngrams": 1835021,
+  "total_ngrams": 243288857
+}

models/tokenizer/af_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2be64d6763f5e6fd310791270e3525ef8484a9db66ef0fff6e0f1476d6a96d34
-size 507153
+oid sha256:1a97f19a8c68005de6f7abc6b7be6ddb4f29dc4bcce8c93bf3ebdb5ef11e7604
+size 507227

models/tokenizer/af_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b5159876118e0a3489a99480419b844198a12a1b816ad232fed8ccae35bb818
-size 786684
+oid sha256:55bba7eb925f3c668030043c5076901224b52cc7e96df7666b413e13212e697e
+size 786319

models/tokenizer/af_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4675b04acf7e973018b38e96d006eb263620e28799e83be7bdd4a27ac0994a13
-size 1361348
+oid sha256:67030a299304ddbe1eff9fd88f0d1dc3f3c5f18aa19ff74bea961a7e3def5ccd
+size 1362130

models/tokenizer/af_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f7dc8da1d29ab51f9c6ac5f659460f17c5222dd813dbe07b43facfc31f68728
-size 371768
+oid sha256:80cf7fe175e99ec4eb884e8d8f7862dfa20f7afa6bfad5e6f90761ad52857495
+size 371773

models/vocabulary/af_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2355921b7485f0850c13db4987f38a9cb0cb18c4d52369c481dbd6c58288967
-size 6384076
+oid sha256:ce902f05737b875d2b4da0e4ba2d6dfd58a9050a2322da999653d4a9bc44ca2e
+size 6405651

models/vocabulary/af_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "af",
-  "vocabulary_size": 403515,
+  "vocabulary_size": 404957,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.02274299800346992,
+    "type_token_ratio": 0.02270829971241366,
     "coverage": {
-      "top_100": 0.4314823858718236,
-      "top_1000": 0.6347620541014498,
-      "top_5000": 0.7838174538213594,
-      "top_10000": 0.8392913943711919
+      "top_100": 0.43161199506831266,
+      "top_1000": 0.6347677515005388,
+      "top_5000": 0.7838429987865031,
+      "top_10000": 0.8392991264448008
     },
-    "hapax_count": 481438,
-    "hapax_ratio": 0.5440266319228253,
-    "total_documents": 126464
+    "hapax_count": 483504,
+    "hapax_ratio": 0.544203966184222,
+    "total_documents": 127286
   }
 }

models/word_markov/af_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:137d7884907248e6dab7f37ec9010eca83d41914f5a04456d97db0173992ed52
-size 77512633
+oid sha256:b49bd37a72d39129642de5b0931091f10c70f359ce73091515163d994205fc0c
+size 77940531

models/word_markov/af_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "af",
-  "unique_contexts": 884548,
-  "total_transitions": 38784545
+  "unique_contexts": 888057,
+  "total_transitions": 38997660
 }

models/word_markov/af_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:509f64b241796227fd01fdc0d6d9738c3943aabf43e5c799d193f8707fa2a669
-size 234719710
+oid sha256:e6097bfa22b20c26b8dae23d259b458ceb0bb2955a518d1a32211556ee0b826a
+size 235747571

models/word_markov/af_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "af",
-  "unique_contexts": 8810967,
-  "total_transitions": 38658081
+  "unique_contexts": 8849236,
+  "total_transitions": 38870374
 }