Upload all models and assets for ceb (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: ceb
-language_name: CEB
+language_name: Cebuano
 language_family: austronesian_philippine_central
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-austronesian_philippine_central
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.023
+    value: 4.059
   - name: best_isotropy
     type: isotropy
-    value: 0.7600
+    value: 0.7670
   - name: vocabulary_size
     type: vocab
-    value: 1000000
-generated: 2025-12-28
+    value: 0
+generated: 2026-01-07
 ---
 
-# CEB - Wikilangs Models
+# Cebuano - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **CEB** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Cebuano** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -44,12 +54,13 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 ### Models & Assets
 
 - Tokenizers (8k, 16k, 32k, 64k)
-- N-gram models (2, 3, 4-gram)
-- Markov chains (context of 1, 2, 3 and 4)
+- N-gram models (2, 3, 4, 5-gram)
+- Markov chains (context of 1, 2, 3, 4 and 5)
 - Subword N-gram and Markov chains
-- Embeddings in various sizes and dimensions
+- Embeddings in various sizes and dimensions (aligned and unaligned)
 - Language Vocabulary
 - Language Statistics
+
 ![Performance Dashboard](visualizations/performance_dashboard.png)
 
 ### Analysis and Evaluation
@@ -59,7 +70,8 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Summary & Recommendations](#6-summary--recommendations)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
+- [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
 
@@ -68,55 +80,57 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 ![Tokenizer Compression](visualizations/tokenizer_compression.png)
 
+![Tokenizer Fertility](visualizations/tokenizer_fertility.png)
+
+![Tokenizer OOV](visualizations/tokenizer_oov.png)
+
+![Total Tokens](visualizations/tokenizer_total_tokens.png)
+
 ### Results
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.166x | 3.08 | 0.4963% | 288,512 |
-| **16k** | 3.546x | 3.45 | 0.5559% | 257,589 |
-| **32k** | 3.786x | 3.68 | 0.5935% | 241,268 |
-| **64k** | 4.023x 🏆 | 3.91 | 0.6307% | 227,048 |
+| **8k** | 3.174x | 3.18 | 0.3878% | 267,679 |
+| **16k** | 3.550x | 3.55 | 0.4338% | 239,262 |
+| **32k** | 3.813x | 3.82 | 0.4660% | 222,758 |
+| **64k** | 4.059x 🏆 | 4.06 | 0.4960% | 209,290 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Ang 1215 (MCCXV) mao ang usa ka tuig sa kalendaryong Hulyano. Ang 1215 maoy usa ...`
+**Sample 1:** `Kahenera sa mga kaka ang Cteniza. Ang Cteniza sakop sa kabanay nga Ctenizidae. A...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁ang ▁ 1 2 1 5 ▁( m cc x ... (+46 more)` | 56 |
-| 16k | `▁ang ▁ 1 2 1 5 ▁( m cc x ... (+39 more)` | 49 |
-| 32k | `▁ang ▁ 1 2 1 5 ▁( m cc x ... (+38 more)` | 48 |
-| 64k | `▁ang ▁ 1 2 1 5 ▁( mcc xv ) ... (+36 more)` | 46 |
+| 8k | `▁kahenera ▁sa ▁mga ▁kaka ▁ang ▁cten iza . ▁ang ▁cten ... (+23 more)` | 33 |
+| 16k | `▁kahenera ▁sa ▁mga ▁kaka ▁ang ▁cten iza . ▁ang ▁cten ... (+22 more)` | 32 |
+| 32k | `▁kahenera ▁sa ▁mga ▁kaka ▁ang ▁cten iza . ▁ang ▁cten ... (+21 more)` | 31 |
+| 64k | `▁kahenera ▁sa ▁mga ▁kaka ▁ang ▁cten iza . ▁ang ▁cten ... (+21 more)` | 31 |
 
-**Sample 2:** `Villy, Ardennes
- Villy, Yonne`
+**Sample 2:** `Ang Jizō-saki ngalan niining mga mosunod: Heyograpiya Hapon Shakaga Hana, punta,...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁v illy , ▁ardennes ▁v illy , ▁yonne` | 8 |
-| 16k | `▁v illy , ▁ardennes ▁v illy , ▁yonne` | 8 |
-| 32k | `▁v illy , ▁ardennes ▁v illy , ▁yonne` | 8 |
-| 64k | `▁v illy , ▁ardennes ▁v illy , ▁yonne` | 8 |
+| 8k | `▁ang ▁j iz ō - s aki ▁ngalan ▁ni in ... (+47 more)` | 57 |
+| 16k | `▁ang ▁j iz ō - s aki ▁ngalan ▁ni ining ... (+36 more)` | 46 |
+| 32k | `▁ang ▁j iz ō - s aki ▁ngalan ▁niining ▁mga ... (+32 more)` | 42 |
+| 64k | `▁ang ▁j iz ō - s aki ▁ngalan ▁niining ▁mga ... (+32 more)` | 42 |
 
-**Sample 3:** `Si Doubleday nga ang hamubong pangalan niini nag tudlo kang:
-
- Henry Doubleday
- ...`
+**Sample 3:** `Ang (MCMLXXXIII) mao ang usa ka tuig sa kalendaryong Gregoryano. Ang maoy usa ka...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁si ▁do ubl ed ay ▁nga ▁ang ▁hamubong ▁pangalan ▁niini ... (+23 more)` | 33 |
-| 16k | `▁si ▁do ubl ed ay ▁nga ▁ang ▁hamubong ▁pangalan ▁niini ... (+23 more)` | 33 |
-| 32k | `▁si ▁do ubl ed ay ▁nga ▁ang ▁hamubong ▁pangalan ▁niini ... (+23 more)` | 33 |
-| 64k | `▁si ▁doubleday ▁nga ▁ang ▁hamubong ▁pangalan ▁niini ▁nag ▁tudlo ▁kang ... (+11 more)` | 21 |
+| 8k | `▁ang ▁( m c m l xx x iii ) ... (+32 more)` | 42 |
+| 16k | `▁ang ▁( m c m l xx x iii ) ... (+28 more)` | 38 |
+| 32k | `▁ang ▁( m c m l xxx iii ) ▁mao ... (+24 more)` | 34 |
+| 64k | `▁ang ▁( mc m l xxx iii ) ▁mao ▁ang ... (+22 more)` | 32 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.023x compression
-- **Lowest UNK Rate:** 8k with 0.4963% unknown tokens
+- **Best Compression:** 64k achieves 4.059x compression
+- **Lowest UNK Rate:** 8k with 0.3878% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -125,57 +139,111 @@ Below are sample sentences tokenized with each vocabulary size:
 
 ![N-gram Perplexity](visualizations/ngram_perplexity.png)
 
+![N-gram Unique](visualizations/ngram_unique.png)
+
 ![N-gram Coverage](visualizations/ngram_coverage.png)
 
 ### Results
 
-| N-gram | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
-|--------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | 2,483 🏆 | 11.28 | 3,473,094 | 35.1% | 79.1% |
-| **2-gram** | 262 🏆 | 8.03 | 32,514 | 66.2% | 99.4% |
-| **3-gram** | 6,019 | 12.56 | 8,522,415 | 25.6% | 70.1% |
-| **3-gram** | 1,492 | 10.54 | 210,233 | 33.0% | 80.5% |
-| **4-gram** | 10,956 | 13.42 | 18,410,507 | 23.0% | 64.2% |
-| **4-gram** | 4,346 | 12.09 | 1,121,715 | 21.0% | 64.2% |
+| N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
+|--------|---------|------------|---------|----------------|------------------|-------------------|
+| **2-gram** | Word | 3,171 | 11.63 | 3,446,236 | 37.4% | 76.3% |
+| **2-gram** | Subword | 218 🏆 | 7.77 | 33,604 | 70.8% | 99.5% |
+| **3-gram** | Word | 6,839 | 12.74 | 7,766,658 | 32.6% | 69.1% |
+| **3-gram** | Subword | 1,277 | 10.32 | 196,868 | 35.6% | 83.3% |
+| **4-gram** | Word | 13,177 | 13.69 | 16,952,568 | 31.0% | 62.8% |
+| **4-gram** | Subword | 3,898 | 11.93 | 1,019,139 | 22.5% | 67.3% |
+| **5-gram** | Word | 19,115 | 14.22 | 18,655,008 | 30.0% | 58.4% |
+| **5-gram** | Subword | 7,890 | 12.95 | 3,628,728 | 16.7% | 59.8% |
 
 ### Top 5 N-grams by Size
 
-**2-grams:**
+**2-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `sa nasod` | 7,048,649 |
+| 2 | `km sa` | 6,204,569 |
+| 3 | `palibot sa` | 5,653,512 |
+| 4 | `ang mga` | 5,645,464 |
+| 5 | `mga gi` | 5,576,920 |
+
+**3-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `mga gi basihan` | 5,576,915 |
+| 2 | `ang mga gi` | 5,576,913 |
+| 3 | `gi basihan niini` | 5,576,912 |
+| 4 | `geonames org cc` | 3,664,283 |
+| 5 | `org cc by` | 3,664,283 |
+
+**4-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `ang mga gi basihan` | 5,576,913 |
+| 2 | `mga gi basihan niini` | 5,576,912 |
+| 3 | `geonames org cc by` | 3,664,283 |
+| 4 | `org cc by post` | 3,664,270 |
+| 5 | `cc by post updated` | 3,664,269 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `ang mga gi basihan niini` | 5,576,912 |
+| 2 | `geonames org cc by post` | 3,664,270 |
+| 3 | `org cc by post updated` | 3,664,269 |
+| 4 | `cc by post updated database` | 3,664,234 |
+| 5 | `post updated database download sa` | 3,664,233 |
+
+**2-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `a _` | 176,572,408 |
+| 2 | `a n` | 170,636,786 |
+| 3 | `n g` | 127,660,424 |
+| 4 | `s a` | 126,044,028 |
+| 5 | `_ s` | 125,029,167 |
+
+**3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `. ang` | 22,734,270 |
-| 2 | `. ,` | 21,897,297 |
-| 3 | `kategoriya :` | 13,463,660 |
-| 4 | `, sa` | 12,991,378 |
-| 5 | `° c` | 8,094,599 |
+| 1 | `_ s a` | 104,157,280 |
+| 2 | `s a _` | 95,124,588 |
+| 3 | `a n g` | 80,898,551 |
+| 4 | `n g _` | 79,824,327 |
+| 5 | `_ a n` | 50,392,535 |
 
-**3-grams:**
+**4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `mga gi basihan` | 5,576,921 |
-| 2 | `ang mga gi` | 5,576,920 |
-| 3 | `gi basihan niini` | 5,576,918 |
-| 4 | `c . ang` | 5,477,045 |
-| 5 | `° c .` | 5,398,278 |
+| 1 | `_ s a _` | 94,060,964 |
+| 2 | `a n g _` | 70,289,894 |
+| 3 | `_ a n g` | 46,728,827 |
+| 4 | `_ n g a` | 28,593,356 |
+| 5 | `n g a _` | 26,245,654 |
 
-**4-grams:**
+**5-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `ang mga gi basihan` | 5,576,919 |
-| 2 | `mga gi basihan niini` | 5,576,918 |
-| 3 | `, sa ° c` | 5,392,546 |
-| 4 | `° c . ang` | 4,999,337 |
-| 5 | `geonames . org (` | 3,664,288 |
+| 1 | `_ a n g _` | 46,539,851 |
+| 2 | `_ n g a _` | 26,090,887 |
+| 3 | `n _ s a _` | 24,592,104 |
+| 4 | `. _ a n g` | 21,317,144 |
+| 5 | `a n g _ k` | 20,331,305 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram with 262
+- **Best Perplexity:** 2-gram (subword) with 218
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~64% of corpus
+- **Coverage:** Top-1000 patterns cover ~60% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -183,55 +251,86 @@ Below are sample sentences tokenized with each vocabulary size:
 
 ![Markov Entropy](visualizations/markov_entropy.png)
 
+![Markov Contexts](visualizations/markov_contexts.png)
+
 ![Markov Branching](visualizations/markov_branching.png)
 
 ### Results
 
-| Context | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
-|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | 1.4458 | 2.724 | 7.69 | 2,678,400 | 0.0% |
-| **1** | 1.9800 | 3.945 | 15.62 | 7,465 | 0.0% |
-| **2** | 0.5102 | 1.424 | 2.56 | 20,310,820 | 49.0% |
-| **2** | 0.7197 | 1.647 | 3.95 | 116,477 | 28.0% |
-| **3** | 0.2244 | 1.168 | 1.69 | 51,557,317 | 77.6% |
-| **3** | 0.6664 | 1.587 | 3.76 | 459,408 | 33.4% |
-| **4** | 0.1061 🏆 | 1.076 | 1.36 | 86,676,271 | 89.4% |
-| **4** | 0.6423 🏆 | 1.561 | 3.28 | 1,725,840 | 35.8% |
+| Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
+|---------|---------|-------------|------------|------------------|-----------------|----------------|
+| **1** | Word | 1.4579 | 2.747 | 8.46 | 2,622,358 | 0.0% |
+| **1** | Subword | 1.5846 | 2.999 | 12.23 | 10,636 | 0.0% |
+| **2** | Word | 0.5081 | 1.422 | 2.51 | 21,964,306 | 49.2% |
+| **2** | Subword | 0.6448 | 1.564 | 3.57 | 129,845 | 35.5% |
+| **3** | Word | 0.2262 | 1.170 | 1.63 | 54,790,128 | 77.4% |
+| **3** | Subword | 0.6034 | 1.519 | 3.47 | 463,245 | 39.7% |
+| **4** | Word | 0.0992 🏆 | 1.071 | 1.32 | 89,104,487 | 90.1% |
+| **4** | Subword | 0.6107 | 1.527 | 3.20 | 1,608,648 | 38.9% |
 
-### Generated Text Samples
+### Generated Text Samples (Word-based)
 
-Below are text samples generated from each Markov chain model:
+Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `. ang kinabugnawan hulyo , sa dagat nga sama niini [ piz alpetta . lepindex :`
-2. `sa ka mga gi basihan niini [ sungai gitam ( cc - by ) ] ;`
-3. `, ug km sa congo aka valley , w . systema dipterorum ( roskov y .`
+1. `sa lintjønnåsen bungtod mikkelhaugen ang poluostrov zuyeva sa amihanan sidlakan dagat kahaboga ang k...`
+2. `ang kinainitan nga matang nga sama niini turkey hill sa british columbia river ang kinahabogang dapi...`
+3. `nga sama niini villabuena del atlántico sur peru nga ugahon ang kinabasaan nga bulan hunyo sa`
 
 **Context Size 2:**
 
-1. `. ang kinahabogang dapit sa mao gihapon nga ngalan , tan - awa ang la laigne sa`
-2. `. , orrell t . , jourbook : entomologist volume : 23 : 76 . yu d`
-3. `kategoriya : mga lungsod sa palibot dunay gihabogon nga ka metro ug km sa sidlakan sa nuaf`
+1. `sa nasod ang klima bugnaw nga ugahon ang kasarangang giiniton c ang kasarangang pag ulan milimetro m...`
+2. `km sa amihanan kasadpan sa washington d c metros ibabaw sa dagat kahaboga ang nahimutangan sa mållok`
+3. `palibot sa desa caringin administratibo nga balangay ang kudumbuwa sa geonames org cc by post update...`
 
 **Context Size 3:**
 
-1. `mga gi basihan niini [ wādī khalīl ] sa [ geonames . org ( cc - by )`
-2. `ang mga gi basihan niini [ lambskär ] sa [ geonames . org ( cc - by )`
-3. `gi basihan niini [ matinsaari ] sa [ geonames . org ( cc - by ) ] ;`
+1. `mga gi basihan niini jessup guymer in austrobaileya 7 15 govaerts r ed for a full list of`
+2. `ang mga gi basihan niini kūh e tīr sa rehiyon palibot sa parksville knob hapit nalukop sa kaumahan`
+3. `gi basihan niini nhamiraze sa geonames org cc by post updated database download sa pahang suba sa ma...`
 
 **Context Size 4:**
 
-1. `ang mga gi basihan niini [ jar - e takhtah chinār ] sa [ geonames . org ( cc`
-2. `mga gi basihan niini [ bustøkniben ] sa [ geonames . org ( cc - by ) ] ;`
-3. `, sa ° c . ang kinainitan nga bulan hunyo , sa milimetro nga ulan , ug ang kinaugahan`
+1. `ang mga gi basihan niini austdalen sa geonames org cc by post updated database download sa suba sa i...`
+2. `mga gi basihan niini cañada del mundo sa dominikanhong republika nahimutang ni sa sentro nga bahin s...`
+3. `geonames org cc by post updated database download sa bungtod sa northern estado sa sudan sa sudan ng...`
+
+
+### Generated Text Samples (Subword-based)
+
+Below are text samples generated from each subword-based Markov chain model:
+
+**Context Size 1:**
+
+1. `_nahinababaes_pi`
+2. `a_mga_nl._sangan`
+3. `nga_mibluagingal`
+
+**Context Size 2:**
+
+1. `a_amasmyctomihapr`
+2. `andsby)];_p.m._an`
+3. `ngaloado_nga_gel.`
+
+**Context Size 3:**
+
+1. `_sa_hayop_sa_tro._`
+2. `sa_orrell_(cc-by)]`
+3. `ang_sourgoin_tom_n`
+
+**Context Size 4:**
+
+1. `_sa_nasod,_km_sa_[_`
+2. `ang_patag_tuig._kin`
+3. `_ang_kinabarat_aaku`
 
 
 ### Key Findings
 
-- **Best Predictability:** Context-4 with 89.4% predictability
+- **Best Predictability:** Context-4 (word) with 90.1% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (1,725,840 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (1,608,648 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -247,64 +346,64 @@ Below are text samples generated from each Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 1,000,000 |
-| Total Tokens | 845,775,833 |
-| Mean Frequency | 845.78 |
-| Median Frequency | 13 |
-| Frequency Std Dev | 118478.53 |
+| Vocabulary Size | 2,197,636 |
+| Total Tokens | 770,818,249 |
+| Mean Frequency | 350.75 |
+| Median Frequency | 6 |
+| Frequency Std Dev | 78759.96 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | sa | 95,127,836 |
-| 2 | ang | 48,190,139 |
-| 3 | nga | 26,124,610 |
-| 4 | mga | 14,558,326 |
-| 5 | kategoriya | 13,463,779 |
-| 6 | ug | 11,614,906 |
-| 7 | c | 9,761,722 |
-| 8 | ni | 8,490,927 |
-| 9 | niini | 7,626,081 |
-| 10 | palibot | 7,306,538 |
+| 1 | sa | 95,123,802 |
+| 2 | ang | 48,189,862 |
+| 3 | nga | 26,091,942 |
+| 4 | ug | 11,614,833 |
+| 5 | mga | 11,196,843 |
+| 6 | c | 9,761,410 |
+| 7 | ni | 8,490,669 |
+| 8 | niini | 7,626,074 |
+| 9 | palibot | 7,306,530 |
+| 10 | nasod | 7,071,533 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | mentang | 8 |
-| 2 | padangalipan | 8 |
-| 3 | palawangan | 8 |
-| 4 | patokoan | 8 |
-| 5 | patongai | 8 |
-| 6 | pentujangan | 8 |
-| 7 | pessailean | 8 |
-| 8 | petampe | 8 |
-| 9 | petape | 8 |
-| 10 | pillong | 8 |
+| 1 | kaliforńijo | 2 |
+| 2 | kaliforniya | 2 |
+| 3 | کیلیفورنیا | 2 |
+| 4 | couzzens | 2 |
+| 5 | hellgrammite | 2 |
+| 6 | powena | 2 |
+| 7 | californië | 2 |
+| 8 | mcgarva | 2 |
+| 9 | fightertown | 2 |
+| 10 | ferril | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 1.4546 |
-| R² (Goodness of Fit) | 0.992518 |
+| Zipf Coefficient | 1.4288 |
+| R² (Goodness of Fit) | 0.993579 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 60.7% |
-| Top 1,000 | 89.2% |
-| Top 5,000 | 93.8% |
-| Top 10,000 | 95.1% |
+| Top 100 | 63.2% |
+| Top 1,000 | 88.4% |
+| Top 5,000 | 93.1% |
+| Top 10,000 | 94.4% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9925 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 60.7% of corpus
-- **Long Tail:** 990,000 words needed for remaining 4.9% coverage
+- **Zipf Compliance:** R²=0.9936 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 63.2% of corpus
+- **Long Tail:** 2,187,636 words needed for remaining 5.6% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -317,24 +416,130 @@ Below are text samples generated from each Markov chain model:
 
 ![t-SNE Sentences](visualizations/tsne_sentences.png)
 
-### Model Comparison
 
-| Model | Vocab Size | Dimension | Avg Norm | Std Norm | Isotropy |
-|-------|------------|-----------|----------|----------|----------|
-| **mono_32d** | 1,545,627 | 32 | 5.888 | 1.742 | 0.7600 🏆 |
-| **mono_64d** | 1,545,627 | 64 | 6.390 | 1.467 | 0.7319 |
-| **mono_128d** | 1,545,627 | 128 | 6.832 | 1.294 | 0.6617 |
-| **embeddings_enhanced** | 0 | 0 | 0.000 | 0.000 | 0.0000 |
+### 5.1 Cross-Lingual Alignment
+
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
+
+
+### 5.2 Model Comparison
+
+| Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
+|-------|-----------|----------|------------------|---------------|----------------|
+| **mono_32d** | 32 | 0.7670 🏆 | 0.3194 | N/A | N/A |
+| **mono_64d** | 64 | 0.7432 | 0.2748 | N/A | N/A |
+| **mono_128d** | 128 | 0.6660 | 0.2423 | N/A | N/A |
+| **aligned_32d** | 32 | 0.7670 | 0.3286 | 0.1020 | 0.4400 |
+| **aligned_64d** | 64 | 0.7432 | 0.2716 | 0.2480 | 0.6140 |
+| **aligned_128d** | 128 | 0.6660 | 0.2452 | 0.3300 | 0.7240 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.7600 (more uniform distribution)
-- **Dimension Trade-off:** Higher dimensions capture more semantics but reduce isotropy
-- **Vocabulary Coverage:** All models cover 1,545,627 words
-- **Recommendation:** 100d for balanced semantic capture and efficiency
+- **Best Isotropy:** mono_32d with 0.7670 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.2803. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 33.0% R@1 in cross-lingual retrieval.
+- **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
-## 6. Summary & Recommendations
+## 6.  Morphological Analysis (Experimental)
+
+This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
+
+### 6.1 Productivity & Complexity
+
+| Metric | Value | Interpretation | Recommendation |
+|--------|-------|----------------|----------------|
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **-0.024** | Low formulaic content | - |
+
+### 6.2 Affix Inventory (Productive Units)
+
+These are the most productive prefixes and suffixes identified by sampling the vocabulary for global substitutability patterns. A unit is considered an affix if stripping it leaves a valid stem that appears in other contexts.
+
+#### Productive Prefixes
+| Prefix | Examples |
+|--------|----------|
+| `-ma` | mazanderanica, magnesita, magnhildmyra |
+
+#### Productive Suffixes
+| Suffix | Examples |
+|--------|----------|
+| `-a` | susumwa, pucanaylla, mazanderanica |
+| `-s` | heteraxinoides, gastroglottis, supersentiens |
+| `-en` | sveinebakken, elgemyrdalen, føytongjen |
+| `-is` | gastroglottis, nooksackensis, naraiensis |
+| `-us` | pseudogymnostreptus, rearedpiaractus, supremus |
+| `-ia` | omphalomia, eugomontia, leucospilaria |
+| `-la` | pucanaylla, diltilla, bulbulla |
+| `-na` | thunbergiana, jajina, coolarrikinna |
+
+### 6.3 Bound Stems (Lexical Roots)
+
+Bound stems are high-frequency subword units that are semantically cohesive but rarely appear as standalone words. These often correspond to the 'core' of a word that requires inflection or derivation to be valid.
+
+| Stem | Cohesion | Substitutability | Examples |
+|------|----------|------------------|----------|
+| `lson` | 2.69x | 160 contexts | olson, alson, elson |
+| `ahim` | 2.83x | 95 contexts | kahim, rahim, tahim |
+| `eona` | 2.74x | 87 contexts | teona, meona, leona |
+| `ngto` | 2.54x | 108 contexts | hangto, singto, langto |
+| `ugna` | 2.37x | 146 contexts | yugna, pugna, ugnat |
+| `ogue` | 2.44x | 115 contexts | bogue, logue, gogue |
+| `etro` | 2.08x | 203 contexts | netro, uetro, etrou |
+| `ands` | 2.06x | 206 contexts | sands, wands, pands |
+| `abaw` | 2.19x | 74 contexts | mabaw, labaw, tabaw |
+| `ecie` | 2.61x | 34 contexts | decie, pecies, specie |
+| `ated` | 2.52x | 37 contexts | dated, rated, hated |
+| `atag` | 1.65x | 256 contexts | atagn, datag, atago |
+
+### 6.4 Affix Compatibility (Co-occurrence)
+
+This table shows which prefixes and suffixes most frequently co-occur on the same stems, revealing the 'stacking' rules of the language's morphology.
+
+| Prefix | Suffix | Frequency | Examples |
+|--------|--------|-----------|----------|
+| `-ma` | `-a` | 56 words | matarrala, mahmudiya |
+| `-ma` | `-s` | 25 words | macrostrobilus, macroconus |
+| `-ma` | `-na` | 13 words | magiana, manvoumouna |
+| `-ma` | `-us` | 9 words | macrostrobilus, macroconus |
+| `-ma` | `-la` | 8 words | matarrala, macunolla |
+| `-ma` | `-is` | 7 words | mallecensis, marizópolis |
+| `-ma` | `-ia` | 4 words | maligia, mariahuslia |
+| `-ma` | `-ra` | 3 words | mautotara, macrochiera |
+| `-ma` | `-en` | 3 words | maben, maureen |
+| `-ma` | `-es` | 2 words | macroscelides, mashes |
+
+### 6.5 Recursive Morpheme Segmentation
+
+Using **Recursive Hierarchical Substitutability**, we decompose complex words into their constituent morphemes. This approach handles nested affixes (e.g., `prefix-prefix-root-suffix`).
+
+| Word | Suggested Split | Confidence | Stem |
+|------|-----------------|------------|------|
+| whittieriana | **`whittier-ia-na`** | 6.0 | `whittier` |
+| darwiniana | **`darwin-ia-na`** | 6.0 | `darwin` |
+| huicumera | **`huicume-ra`** | 4.5 | `huicume` |
+| javorkana | **`javorka-na`** | 4.5 | `javorka` |
+| olavsbekken | **`olavsbekk-en`** | 4.5 | `olavsbekk` |
+| campelles | **`campell-es`** | 4.5 | `campell` |
+| apolinaria | **`apolinar-ia`** | 4.5 | `apolinar` |
+| steyskalia | **`steyskal-ia`** | 4.5 | `steyskal` |
+| liniholmen | **`liniholm-en`** | 4.5 | `liniholm` |
+| finngrunden | **`finngrund-en`** | 4.5 | `finngrund` |
+| maaprobahan | **`ma-aprobahan`** | 4.5 | `aprobahan` |
+| macrostylospora | **`ma-crostylospo-ra`** | 3.0 | `crostylospo` |
+| saharolana | **`saharo-la-na`** | 3.0 | `saharo` |
+| maxwellensis | **`ma-xwellens-is`** | 3.0 | `xwellens` |
+| mappianthus | **`ma-ppianth-us`** | 3.0 | `ppianth` |
+
+### 6.6 Linguistic Interpretation
+
+> **Automated Insight:**
+The language Cebuano shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
+
+---
+## 7. Summary & Recommendations
 
 ![Performance Dashboard](visualizations/performance_dashboard.png)
 
@@ -342,11 +547,12 @@ Below are text samples generated from each Markov chain model:
 
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
-| Tokenizer | **32k BPE** | Best compression (4.02x) with low UNK rate |
-| N-gram | **5-gram** | Lowest perplexity (262) |
-| Markov | **Context-4** | Highest predictability (89.4%) |
+| Tokenizer | **64k BPE** | Best compression (4.06x) |
+| N-gram | **2-gram** | Lowest perplexity (218) |
+| Markov | **Context-4** | Highest predictability (90.1%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
+
 ---
 ## Appendix: Metrics Glossary & Interpretation Guide
 
@@ -536,7 +742,8 @@ If you use these models in your research, please cite:
   author = {Kamali, Omar},
   title = {Wikilangs: Open NLP Models for Wikipedia Languages},
   year = {2025},
-  publisher = {HuggingFace},
+  doi = {10.5281/zenodo.18073153},
+  publisher = {Zenodo},
   url = {https://huggingface.co/wikilangs}
   institution = {Omneity Labs}
 }
@@ -552,7 +759,8 @@ MIT License - Free for academic and commercial use.
 - 🤗 Models: [huggingface.co/wikilangs](https://huggingface.co/wikilangs)
 - 📊 Data: [wikipedia-monthly](https://huggingface.co/datasets/omarkamali/wikipedia-monthly)
 - 👤 Author: [Omar Kamali](https://huggingface.co/omarkamali)
+- 🤝 Sponsor: [Featherless AI](https://featherless.ai)
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2025-12-28 22:39:27*
+*Report Date: 2026-01-07 20:10:38*

models/embeddings/aligned/ceb_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3eb8cd36ef12787b52a4668ec8d932c15ae2506ba3d10269c67067feb027418f
+size 2541152375

models/embeddings/aligned/ceb_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "ceb", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/ceb_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea5b20fdaac8234f50ae2c532a86b279e3aff39e4d73f285b1e0a70647b71f91
+size 65664

models/embeddings/aligned/ceb_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "ceb",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 152433,
+  "vocab_size": 1454790
+}

models/embeddings/aligned/ceb_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eafa8758d8b2648080dbb701fc676f2b6825d9fc3b9caa5a358d6a42f7c68238
+size 655873655

models/embeddings/aligned/ceb_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "ceb", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/ceb_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94bb2c9dcb6b024b0a37da5310b8164eaccd4a8d06893cf45a28f70e9ed38b1d
+size 4224

models/embeddings/aligned/ceb_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "ceb",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 152433,
+  "vocab_size": 1454790
+}

models/embeddings/aligned/ceb_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a051326edada076b29c9b2bc4a1c13e1b92c0e582e710fcf349f23daf828fb19
+size 1284299895

models/embeddings/aligned/ceb_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "ceb", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/ceb_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c352c1836ef6cb3518cfb8cf48cd40d6d99ca83e50dedbf9849d9518e0efe65
+size 16512

models/embeddings/aligned/ceb_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "ceb",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 152433,
+  "vocab_size": 1454790
+}

models/embeddings/monolingual/ceb_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:42aab08af6a9effecb1dd1a8daddf5394c4fa5c310fb4f82f9d1b96c78267925
-size 2636678209
+oid sha256:3eb8cd36ef12787b52a4668ec8d932c15ae2506ba3d10269c67067feb027418f
+size 2541152375

models/embeddings/monolingual/ceb_128d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 128,
   "version": "monolingual",
   "training_params": {
-    "dim": 128,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 128
   },
-  "vocab_size": 1545627
+  "vocab_size": 1454790
 }

models/embeddings/monolingual/ceb_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f87fe735c27ef9aafdff64e210b5e4684ea25d1e3dbbc3d6b511fc677fc903d
-size 681636673
+oid sha256:eafa8758d8b2648080dbb701fc676f2b6825d9fc3b9caa5a358d6a42f7c68238
+size 655873655

models/embeddings/monolingual/ceb_32d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 32,
   "version": "monolingual",
   "training_params": {
-    "dim": 32,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 32
   },
-  "vocab_size": 1545627
+  "vocab_size": 1454790
 }

models/embeddings/monolingual/ceb_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f23bb6c3132f01e2dffd96a4b5bc7e374ddbb69ca8edb7b8f0fa90dc6a542e52
-size 1333317185
+oid sha256:a051326edada076b29c9b2bc4a1c13e1b92c0e582e710fcf349f23daf828fb19
+size 1284299895

models/embeddings/monolingual/ceb_64d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 64,
   "version": "monolingual",
   "training_params": {
-    "dim": 64,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 64
   },
-  "vocab_size": 1545627
+  "vocab_size": 1454790
 }

models/subword_markov/ceb_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75eee096310049e7245bcee5d8d44990052e6f878d91fb14d82179c90ec1be30
-size 702190
+oid sha256:4f768ccf3c1b17ee6c3155af7196c8d3225ee1b5b17f295b763044d932178548
+size 843230

models/subword_markov/ceb_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "ceb",
-  "unique_contexts": 7465,
-  "total_transitions": 5363634377
+  "unique_contexts": 10636,
+  "total_transitions": 4741280452
 }

models/subword_markov/ceb_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da081160892c12c55ca085c74a7860d55ebf73556a941f29f3ea6ee7f4261eba
-size 3994591
+oid sha256:740d6da8735c5fe348a7d02eb110099031a0c7520aa307d58faedb1d948abea4
+size 4133523

models/subword_markov/ceb_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "ceb",
-  "unique_contexts": 116477,
-  "total_transitions": 5357518115
+  "unique_contexts": 129845,
+  "total_transitions": 4735165236
 }

models/subword_markov/ceb_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e89ee5c77a03fb98bfc85c9b8338ba70da0ee7bd637bf3e952dba101194c58bb
-size 16258109
+oid sha256:7280cd84fb22bd825ecd5918fd659b4ee44d894dd426f7e4863f41140c3b9065
+size 15714134

models/subword_markov/ceb_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "ceb",
-  "unique_contexts": 459408,
-  "total_transitions": 5351401853
+  "unique_contexts": 463245,
+  "total_transitions": 4729050020
 }

models/subword_markov/ceb_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa1afce11a5001cd502a857781285516d4cd5c3f72720239d8a9181ed19c48a0
-size 54302303
+oid sha256:c541cfbd127565d46008005e601455b973d3ebc5d5be3c000d9b4177c0faf931
+size 50278691

models/subword_markov/ceb_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "ceb",
-  "unique_contexts": 1725840,
-  "total_transitions": 5345285591
+  "unique_contexts": 1608648,
+  "total_transitions": 4722934804
 }

models/subword_ngram/ceb_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ae4ef8a8dc6cccad7b03f52ba1f8907812c1874ab0299b5e2eccbd41b6830b2
-size 448161
+oid sha256:75effbc8c3c560b65d736c9461a76644273c13e5a1bdf333accf36f01b482fd9
+size 472677

models/subword_ngram/ceb_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "ceb",
-  "unique_ngrams": 32514,
-  "total_ngrams": 5363634377
+  "unique_ngrams": 33604,
+  "total_ngrams": 4741280452
 }

models/subword_ngram/ceb_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b5f2298d782051f5882d10b41d5391ea25fc7c33b4c9f0787addc7efa452f93
-size 2626190
+oid sha256:bf35e60b92e9a30a6bddca322cbb5e9b26d7208c3b985403b5ad20e71d81c489
+size 2514513

models/subword_ngram/ceb_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "ceb",
-  "unique_ngrams": 210233,
-  "total_ngrams": 5357518115
+  "unique_ngrams": 196868,
+  "total_ngrams": 4735165236
 }

models/subword_ngram/ceb_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f47830959cc3a6165b807813b29917389d27c20cab9f9f85ea225d9df7c66211
-size 13858488
+oid sha256:c8bed27ef2e0fd850c1d5afdae26a0c1d582caa13740b70d686a3c4b6572dfe6
+size 12745233

models/subword_ngram/ceb_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "ceb",
-  "unique_ngrams": 1121715,
-  "total_ngrams": 5351401853
+  "unique_ngrams": 1019139,
+  "total_ngrams": 4729050020
 }

models/subword_ngram/ceb_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ded5fa72b1333db15722dc0eadc2671d488350d645d3763e7c475a650a835795
+size 44204990

models/subword_ngram/ceb_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "ceb",
+  "unique_ngrams": 3628728,
+  "total_ngrams": 4722934804
+}

models/tokenizer/ceb_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:603bba5792541d1e0e40f62a0807a06e1ecdd93face5666eb5cc7669fe8f92b3
-size 507882
+oid sha256:d0524fd2b40acc6f5b34cb83b12b24872b8c64abbd29e4a958043cd2c376fce7
+size 510736

models/tokenizer/ceb_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:86f7da465475641e9241e82d2fb8af8119bc8bc35a96b01ec68c92816a70efa6
-size 793102
+oid sha256:388d9039e50faf9eeb7281c832737aadf1e832bbe627c92b606b3ed690ea18f8
+size 798937

models/tokenizer/ceb_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c578be212d9f5b72edbc15bab7a6bed92723608475c4b47a09a01fdc7b14a04
-size 1380555
+oid sha256:0eaa647544e23d6b15bfee9173d40a3962bae64e5745da112bfba70081de1b77
+size 1394915

models/tokenizer/ceb_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:093777627391419d288ec95c764e3df10df5f26a9431ae0920f42ce278cbd9b7
-size 371978
+oid sha256:074f697a2540d914dbc182aa0ad66124f9535c265b7cfad75e78a6a21b055930
+size 373160

models/vocabulary/ceb_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f95b64c94d5bfea3ae0b3adfe22a45a4b55ec4e76158c85816dabaad27ce559d
-size 14409016
+oid sha256:4393f136f1e1677ad29b26e914ac0063db786da099d2088540d60a7c8d0a9015
+size 30923898

models/vocabulary/ceb_vocabulary_metadata.json

@@ -1,16 +1,17 @@
 {
   "language": "ceb",
-  "vocabulary_size": 1000000,
+  "vocabulary_size": 2197636,
+  "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.00314743750932242,
+    "type_token_ratio": 0.0034010600533183766,
     "coverage": {
-      "top_100": 0.6036295391408708,
-      "top_1000": 0.8863501456542678,
-      "top_5000": 0.9327801826871985,
-      "top_10000": 0.9454672256327237
+      "top_100": 0.6313898005092059,
+      "top_1000": 0.8839513376148225,
+      "top_5000": 0.9301546296408513,
+      "top_10000": 0.9430208060874313
     },
-    "hapax_count": 461125,
-    "hapax_ratio": 0.1721708442559998,
-    "total_documents": 6116262
+    "hapax_count": 425410,
+    "hapax_ratio": 0.1621816773323838,
+    "total_documents": 6115216
   }
 }

models/vocabulary/ceb_vocabulary_top.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b883ab7c42fc9a39c0b762affb65bd56f057e451df5cff5a41c183ddac4fad1e
+size 14413163

models/vocabulary/ceb_vocabulary_top_metadata.json

@@ -0,0 +1,20 @@
+{
+  "language": "ceb",
+  "vocabulary_size": 1000000,
+  "variant": "top",
+  "statistics": {
+    "type_token_ratio": 0.0034010600533183766,
+    "coverage": {
+      "top_100": 0.6313898005092059,
+      "top_1000": 0.8839513376148225,
+      "top_5000": 0.9301546296408513,
+      "top_10000": 0.9430208060874313
+    },
+    "hapax_count": 425410,
+    "hapax_ratio": 0.1621816773323838,
+    "total_documents": 6115216,
+    "top_vocab_size": 1000000,
+    "coverage_ratio": 0.9935723581748113,
+    "tokens_excluded": 1197636
+  }
+}

models/word_markov/ceb_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e44c59f28e8285985bf24534faa2a26ecb1297f68104a8b5770f2dbb3cb5e6c8
-size 163469464
+oid sha256:8d44f8a21899c99ba775bbcd5e8e5e784ac724f54d572a7815600b0965968b7e
+size 175693160

models/word_markov/ceb_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "ceb",
-  "unique_contexts": 2678400,
-  "total_transitions": 1160169556
+  "unique_contexts": 2622358,
+  "total_transitions": 765128443
 }