Upload all models and assets for dv (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: dv
-language_name: DV
+language_name: Divehi
 language_family: indoaryan_insular
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-indoaryan_insular
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 6.114
+    value: 5.583
   - name: best_isotropy
     type: isotropy
-    value: 0.8870
+    value: 0.8795
   - name: vocabulary_size
     type: vocab
-    value: 6434
-generated: 2025-12-30
+    value: 0
+generated: 2026-01-04
 ---
 
-# DV - Wikilangs Models
+# Divehi - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **DV** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Divehi** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -44,12 +54,13 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 ### Models & Assets
 
 - Tokenizers (8k, 16k, 32k, 64k)
-- N-gram models (2, 3, 4-gram)
-- Markov chains (context of 1, 2, 3 and 4)
+- N-gram models (2, 3, 4, 5-gram)
+- Markov chains (context of 1, 2, 3, 4 and 5)
 - Subword N-gram and Markov chains
-- Embeddings in various sizes and dimensions
+- Embeddings in various sizes and dimensions (aligned and unaligned)
 - Language Vocabulary
 - Language Statistics
+
 ![Performance Dashboard](visualizations/performance_dashboard.png)
 
 ### Analysis and Evaluation
@@ -59,7 +70,8 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Summary & Recommendations](#6-summary--recommendations)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
+- [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
 
@@ -68,53 +80,57 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 ![Tokenizer Compression](visualizations/tokenizer_compression.png)
 
+![Tokenizer Fertility](visualizations/tokenizer_fertility.png)
+
+![Tokenizer OOV](visualizations/tokenizer_oov.png)
+
+![Total Tokens](visualizations/tokenizer_total_tokens.png)
+
 ### Results
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 4.416x | 4.38 | 0.0656% | 543,835 |
-| **16k** | 5.055x | 5.01 | 0.0751% | 475,116 |
-| **32k** | 5.621x | 5.58 | 0.0836% | 427,269 |
-| **64k** | 6.114x 🏆 | 6.06 | 0.0909% | 392,821 |
+| **8k** | 4.195x | 4.20 | 0.4815% | 567,427 |
+| **16k** | 4.753x | 4.76 | 0.5455% | 500,811 |
+| **32k** | 5.229x | 5.24 | 0.6001% | 455,260 |
+| **64k** | 5.583x 🏆 | 5.59 | 0.6407% | 426,395 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `ވާރުތަ ބަލި އަކީ ލޭގެ ގުޅުމުގެ ސަބަބުން ދަރިފަސްކޮޅަށް ދެމިގެންދާ ބަލިތަކެވެ.
-
-ޤ...`
+**Sample 1:** `ޅ.އަތޮޅު ތަޢުލީމީ މަރުކަޒަކީ ޅ. ހިންނަވަރުގައި ހުންނަ މަދަރުސާ އެކެވެ. ސްކޫލުތައ...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁ވާރު ތަ ▁ބަލި ▁އަކީ ▁ލޭގެ ▁ގުޅ ުމުގެ ▁ސަބަބުން ▁ދަރިފ ަސް ... (+9 more)` | 19 |
-| 16k | `▁ވާރުތަ ▁ބަލި ▁އަކީ ▁ލޭގެ ▁ގުޅުމުގެ ▁ސަބަބުން ▁ދަރިފަސް ކޮޅަށް ▁ދެމިގެން ދާ ... (+6 more)` | 16 |
-| 32k | `▁ވާރުތަ ▁ބަލި ▁އަކީ ▁ލޭގެ ▁ގުޅުމުގެ ▁ސަބަބުން ▁ދަރިފަސް ކޮޅަށް ▁ދެމިގެންދާ ▁ބަލިތަކެވެ ... (+4 more)` | 14 |
-| 64k | `▁ވާރުތަ ▁ބަލި ▁އަކީ ▁ލޭގެ ▁ގުޅުމުގެ ▁ސަބަބުން ▁ދަރިފަސް ކޮޅަށް ▁ދެމިގެންދާ ▁ބަލިތަކެވެ ... (+4 more)` | 14 |
+| 8k | `▁ޅ . އަތޮޅު ▁ތަޢުލީމީ ▁މަރުކަޒ ަކީ ▁ޅ . ▁ހިން ނ ... (+7 more)` | 17 |
+| 16k | `▁ޅ . އަތޮޅު ▁ތަޢުލީމީ ▁މަރުކަޒަކީ ▁ޅ . ▁ހިން ނ ަވަރު ... (+6 more)` | 16 |
+| 32k | `▁ޅ . އަތޮޅު ▁ތަޢުލީމީ ▁މަރުކަޒަކީ ▁ޅ . ▁ހި��ްނ ަވަރު ގައި ... (+5 more)` | 15 |
+| 64k | `▁ޅ . އަތޮޅު ▁ތަޢުލީމީ ▁މަރުކަޒަކީ ▁ޅ . ▁ހިންނަވަރުގައި ▁ހުންނަ ▁މަދަރުސާ ... (+3 more)` | 13 |
 
-**Sample 2:** `މިއީ 20ވަނަ ޤަރުނުގެ 99ވަނަ އަހަރެވެ.`
+**Sample 2:** `ނިކަކޯޅި ބަވާސީ އަކީ ނިކަކޯޅިއެއްގެ ސިފައިގައި ފުރަގަސް ފަރާތުން ނިކުންނަ ބައްޔެ...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁މިއީ ▁ 2 0 ވަނަ ▁ޤަރުނުގެ ▁ 9 9 ވަނަ ... (+2 more)` | 12 |
-| 16k | `▁މިއީ ▁ 2 0 ވަނަ ▁ޤަރުނުގެ ▁ 9 9 ވަނަ ... (+2 more)` | 12 |
-| 32k | `▁މިއީ ▁ 2 0 ވަނަ ▁ޤަރުނުގެ ▁ 9 9 ވަނަ ... (+2 more)` | 12 |
-| 64k | `▁މިއީ ▁ 2 0 ވަނަ ▁ޤަރުނުގެ ▁ 9 9 ވަނަ ... (+2 more)` | 12 |
+| 8k | `▁ނިކ ަކޯ ޅި ▁ބ ަވާ ސީ ▁އަކީ ▁ނިކ ަކޯ ޅ ... (+9 more)` | 19 |
+| 16k | `▁ނިކ ަކޯޅި ▁ބ ަވާ ސީ ▁އަކީ ▁ނިކ ަކޯ ޅ ިއެއްގެ ... (+6 more)` | 16 |
+| 32k | `▁ނިކ ަކޯޅި ▁ބަވާސީ ▁އަކީ ▁ނިކ ަކޯ ޅިއެއްގެ ▁ސިފައިގައި ▁ފުރަގަސް ▁ފަރާތުން ... (+3 more)` | 13 |
+| 64k | `▁ނިކަކޯޅި ▁ބަވާސީ ▁އަކީ ▁ނިކަކޯޅިއެއްގެ ▁ސިފައިގައި ▁ފުރަގަސް ▁ފަރާތުން ▁ނިކުންނަ ▁ބައްޔެކެވެ .` | 10 |
 
-**Sample 3:** `މި މަޒުމޫނަކީ ނައިޖީރިއާގެ ނާއިބު ރައީސް އާ ބެހޭ މަޒުމޫނެކެވެ.`
+**Sample 3:** `ފައިފެޅުން އަކީ ބައްޔެއްގެ ސަބަބުން ފައިގެ ހުދުހަން އެކި ދިމަދމާލުން ކެނޑުމެވެ.`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁މި ▁މަޒުމޫނ ަކީ ▁ނައިޖީ ރިއާގެ ▁ނާއިބު ▁ރައީސް ▁އާ ▁ބެހޭ ▁މަޒުމޫނ ... (+2 more)` | 12 |
-| 16k | `▁މި ▁މަޒުމޫނ ަކީ ▁ނައިޖީ ރިއާގެ ▁ނާއިބު ▁ރައީސް ▁އާ ▁ބެހޭ ▁މަޒުމޫނ ... (+2 more)` | 12 |
-| 32k | `▁މި ▁މަޒުމޫނަކީ ▁ނައިޖީރިއާގެ ▁ނާއިބު ▁ރައީސް ▁އާ ▁ބެހޭ ▁މަޒުމޫނެކެވެ .` | 9 |
-| 64k | `▁މި ▁މަޒުމޫނަކީ ▁ނައިޖީރިއާގެ ▁ނާއިބު ▁ރައީސް ▁އާ ▁ބެހޭ ▁މަޒުމޫނެކެވެ .` | 9 |
+| 8k | `▁ފައި ފ ެޅ ުން ▁އަކީ ▁ބައްޔެއްގެ ▁ސަބަބުން ▁ފައިގެ ▁ހުދ ުހ ... (+9 more)` | 19 |
+| 16k | `▁ފައި ފ ެޅުން ▁އަކީ ▁ބައްޔެއްގެ ▁ސަބަބުން ▁ފައިގެ ▁ހުދ ުހ ަން ... (+8 more)` | 18 |
+| 32k | `▁ފައިފ ެޅުން ▁އަކީ ▁ބައްޔެއްގެ ▁ސަބަބުން ▁ފައިގެ ▁ހުދުހ ަން ▁އެކި ▁ދިމަދ ... (+4 more)` | 14 |
+| 64k | `▁ފައިފެޅުން ▁އަކީ ▁ބައްޔެއްގެ ▁ސަބަބުން ▁ފައިގެ ▁ހުދުހަން ▁އެކި ▁ދިމަދމާލުން ▁ކެނޑުމެވެ .` | 10 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 6.114x compression
-- **Lowest UNK Rate:** 8k with 0.0656% unknown tokens
+- **Best Compression:** 64k achieves 5.583x compression
+- **Lowest UNK Rate:** 8k with 0.4815% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -123,57 +139,111 @@ Below are sample sentences tokenized with each vocabulary size:
 
 ![N-gram Perplexity](visualizations/ngram_perplexity.png)
 
+![N-gram Unique](visualizations/ngram_unique.png)
+
 ![N-gram Coverage](visualizations/ngram_coverage.png)
 
 ### Results
 
-| N-gram | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
-|--------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | 280 🏆 | 8.13 | 6,213 | 66.8% | 98.6% |
-| **2-gram** | 309 🏆 | 8.27 | 4,183 | 65.1% | 98.3% |
-| **3-gram** | 1,977 | 10.95 | 27,387 | 28.4% | 74.6% |
-| **3-gram** | 2,024 | 10.98 | 28,100 | 30.0% | 74.0% |
-| **4-gram** | 9,545 | 13.22 | 108,940 | 14.2% | 44.6% |
-| **4-gram** | 8,722 | 13.09 | 111,102 | 15.8% | 46.9% |
+| N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
+|--------|---------|------------|---------|----------------|------------------|-------------------|
+| **2-gram** | Word | 10,033 | 13.29 | 18,085 | 11.2% | 34.3% |
+| **2-gram** | Subword | 1,740 🏆 | 10.76 | 17,306 | 35.4% | 73.1% |
+| **3-gram** | Word | 12,820 | 13.65 | 22,046 | 10.8% | 30.6% |
+| **3-gram** | Subword | 11,965 | 13.55 | 83,683 | 14.8% | 40.7% |
+| **4-gram** | Word | 44,408 | 15.44 | 64,258 | 6.5% | 16.2% |
+| **4-gram** | Subword | 47,194 | 15.53 | 264,508 | 8.4% | 24.1% |
+| **5-gram** | Word | 40,713 | 15.31 | 56,606 | 6.9% | 15.7% |
+| **5-gram** | Subword | 104,406 | 16.67 | 409,837 | 5.5% | 16.8% |
 
 ### Top 5 N-grams by Size
 
-**2-grams:**
+**2-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `ވަނަ އަހަރު` | 1,832 |
+| 2 | `ނުވަތަ އަކީ` | 707 |
+| 3 | `ވަނަ އަހަރުގެ` | 673 |
+| 4 | `ވަނަ ދުވަހެވެ` | 616 |
+| 5 | `މީގެ އިތުރުން` | 596 |
+
+**3-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `އަކީ މީލާދީ ކަލަންޑަރުގެ` | 375 |
+| 2 | `ދުވަސްތަކާއި ފާހަގަ ކުރެވޭ` | 364 |
+| 3 | `ބަންދު ދުވަސްތަކާއި ފާހަގަ` | 364 |
+| 4 | `ފާހަގަ ކުރެވޭ ދުވަހެއްގެ` | 364 |
+| 5 | `ކުރެވޭ ދުވަހެއްގެ ގޮތުގައި` | 364 |
+
+**4-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `ފާހަގަ ކުރެވޭ ދުވަހެއްގެ ގޮތުގައި` | 364 |
+| 2 | `ދުވަސްތަކާއި ފާހަގަ ކުރެވޭ ދުވަހެއްގެ` | 364 |
+| 3 | `ބަންދު ދުވަސްތަކާއި ފާހަގަ ކުރެވޭ` | 364 |
+| 4 | `އުފަންވި މީހުން މަރުވި މީހުން` | 349 |
+| 5 | `މީހުން ބަންދު ދުވަސްތަކާއި ފާހަގަ` | 340 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `ބަންދު ދުވަސްތަކާއި ފާހަގަ ކުރެވޭ ދުވަހެއްގެ` | 364 |
+| 2 | `ދުވަސްތަކާއި ފާހަގަ ކުރެވޭ ދުވަހެއްގެ ގޮތުގައި` | 364 |
+| 3 | `މީހުން ބަންދު ދުވަސްތަކާއި ފާހަގަ ކުރެވޭ` | 340 |
+| 4 | `މަރުވި މީހުން ބަންދު ދުވަސްތަކާއި ފާހަގަ` | 339 |
+| 5 | `މީހުން މަރުވި މީހުން ބަންދު ދުވަސްތަކާއި` | 329 |
+
+**2-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `ން _` | 90,135 |
+| 2 | `ގެ _` | 83,101 |
+| 3 | `. _` | 66,551 |
+| 4 | `ވެ .` | 64,305 |
+| 5 | `އި _` | 60,871 |
+
+**3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `ނ ް` | 201,447 |
-| 2 | `ަ އ` | 180,556 |
-| 3 | `އ ި` | 156,059 |
-| 4 | `ވ ެ` | 109,701 |
-| 5 | `އ ް` | 108,512 |
+| 1 | `ވެ . _` | 61,497 |
+| 2 | `އެ ވެ .` | 36,492 |
+| 3 | `ގަ އި _` | 36,034 |
+| 4 | `ތަ އް _` | 10,452 |
+| 5 | `ކެ ވެ .` | 10,355 |
 
-**3-grams:**
+**4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `ަ އ ި` | 98,210 |
-| 2 | `ެ ވ ެ` | 69,037 |
-| 3 | `ު ނ ް` | 66,257 |
-| 4 | `ވ ެ .` | 64,917 |
-| 5 | `ަ ށ ް` | 49,257 |
+| 1 | `އެ ވެ . _` | 35,128 |
+| 2 | `ކެ ވެ . _` | 9,815 |
+| 3 | `_ އަ ދި _` | 9,086 |
+| 4 | `ވެ . _ މި` | 8,503 |
+| 5 | `ވެ . _ އެ` | 6,652 |
 
-**4-grams:**
+**5-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `ެ ވ ެ .` | 64,897 |
-| 2 | `ގ ަ އ ި` | 40,722 |
-| 3 | `އ ެ ވ ެ` | 37,057 |
-| 4 | `ު ގ ަ އ` | 23,243 |
-| 5 | `އ ި ނ ް` | 18,356 |
+| 1 | `_ އެ ވެ . _` | 6,310 |
+| 2 | `ވެ އެ ވެ . _` | 5,392 |
+| 3 | `ގަ އެ ވެ . _` | 4,655 |
+| 4 | `_ އެ ން މެ _` | 4,586 |
+| 5 | `އެ ވެ . _ މި` | 4,463 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram with 280
+- **Best Perplexity:** 2-gram (subword) with 1,740
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~47% of corpus
+- **Coverage:** Top-1000 patterns cover ~17% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -181,55 +251,86 @@ Below are sample sentences tokenized with each vocabulary size:
 
 ![Markov Entropy](visualizations/markov_entropy.png)
 
+![Markov Contexts](visualizations/markov_contexts.png)
+
 ![Markov Branching](visualizations/markov_branching.png)
 
 ### Results
 
-| Context | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
-|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | 0.5858 | 1.501 | 3.83 | 15,381 | 41.4% |
-| **1** | 1.1454 | 2.212 | 8.73 | 1,139 | 0.0% |
-| **2** | 0.2988 🏆 | 1.230 | 2.08 | 58,771 | 70.1% |
-| **2** | 0.9952 🏆 | 1.993 | 5.52 | 9,934 | 0.5% |
-| **3** | 0.3330 | 1.260 | 2.10 | 122,099 | 66.7% |
-| **3** | 0.7840 | 1.722 | 3.51 | 54,858 | 21.6% |
-| **4** | 0.3715 | 1.294 | 2.01 | 255,795 | 62.9% |
-| **4** | 0.5399 | 1.454 | 2.34 | 192,399 | 46.0% |
+| Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
+|---------|---------|-------------|------------|------------------|-----------------|----------------|
+| **1** | Word | 0.7502 | 1.682 | 4.34 | 120,955 | 25.0% |
+| **1** | Subword | 1.3036 | 2.468 | 18.11 | 2,104 | 0.0% |
+| **2** | Word | 0.1780 | 1.131 | 1.33 | 523,452 | 82.2% |
+| **2** | Subword | 0.8357 | 1.785 | 4.91 | 38,101 | 16.4% |
+| **3** | Word | 0.0519 | 1.037 | 1.08 | 692,308 | 94.8% |
+| **3** | Subword | 0.5690 | 1.484 | 2.88 | 187,098 | 43.1% |
+| **4** | Word | 0.0200 🏆 | 1.014 | 1.03 | 741,793 | 98.0% |
+| **4** | Subword | 0.3828 | 1.304 | 1.92 | 538,145 | 61.7% |
 
-### Generated Text Samples
+### Generated Text Samples (Word-based)
 
-Below are text samples generated from each Markov chain model:
+Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `ަ ގ ެ ކ ޫ ގ ެ . 1980މ . އ ް ނ ާ ނ ީ`
-2. `ް ނ ް ފ ަ ބ ަ ކ ު ރ ި ތ ަ ރ ަ ސ`
-3. `ެ . މ ަ ބ ޮ ހ ު ތ ަ ލ ޭ އ ި ހ ެ`
+1. `އަދި ބެންގާޅީ ފިލްމްތަކުގައި އެބަޔަކާއެކު ނ އަތޮޅުގައި މީހުން މަރުވި މީހުން ވިހަނީ ފަންސަވީސް އަހަރާ...`
+2. `އެވެ ސިސްޓެމިކް ލޫޕަސް އެރިތެމަޓޯސަސް ގެ ނަންދެވުނު މަޝްހޫރު ބުދު ހަރުކުރުމަށް ތައްޔާރު ކުރައްވައިގެ...`
+3. `އަކީ ޢަރަބީންގެ ގާތުގައި މިއީ ދުނިޔޭގައި 58 ވަނަ އަހަރާ ހަމައަށް މަސައްކަތްކުރައްވައިފައި ވަނީ އަމުރ...`
 
 **Context Size 2:**
 
-1. `ނ ް ނ ު ކ ޮ ޅ ު ތ ު ރ ަ އ ި ބ ަ ބ`
-2. `ަ އ ް ކ ަ ނ ް ފ ަ ހ ަ އ ި ވ ެ . މ`
-3. `އ ި ގ ެ ލ ަ ބ ޫ ވ ާ ކ ަ ށ ް އ ި ހ`
+1. `ވަނަ އަހަރު ފެކަލްޓީ އޮފް އިންޖިނިއަރިންގ އެންޑް ޓެކްނޮލޮޖީ އާރްޔޫއީޓީ ސައިޚް މުޖީބުރު ރަޙްމާން ބަން...`
+2. `ނުވަތަ އަކީ މިޔަރުގެ ވައްތަރެކެވެ މިއީ އަތޮޅުން ބޭރުގައި ކުރާ ލޭނުގެ މަސްވެރިކަމުގައެވެ މިމަސް އެންމ...`
+3. `ވަނަ އަހަރުގެ ބޯހިމެނުމުގެ ނަތީޖާތައް ދައްކާގޮތުން މާޅޮސްމަޑުލު އުތުރުބުރީގެ އާބާދީ އިތުރުވަމުން ދިއ...`
 
 **Context Size 3:**
 
-1. `ަ އ ި ވ ާ ހ ަ ށ ި ގ ަ ނޑ ު ގ ަ އ ި ނ`
-2. `ެ ވ ެ . ފ ި ލ ް ރ ަ ސ ް ޖ ެ ހ ޭ ވ ަ`
-3. `ު ނ ް ކ ު ރ ެ އ ް ވ ާ ފ ަ އ ި ވ ާ ޔ`
+1. `އަކީ މީލާދީ ކަލަންޑަރުގެ 146 ވަނަ ދުވަހެވެ ޙާދިސާތައް އުފަންވި މީހުން މަރުވި މީހުން ބަންދު ދުވަސްތަކ...`
+2. `ދުވަސްތަކާއި ފާހަގަ ކުރެވޭ ދުވަހެއްގެ ގޮތުގައި ދިވެހިރާއްޖެ މަސްވެރިންގެ ދުވަސް`
+3. `ބަންދު ދުވަސްތަކާއި ފާހަގަ ކުރެވޭ ދުވަހެއްގެ ގޮތުގައި ނޯވޭ ޔުނިއަން ޑިސޮލިއުޝަން ޑޭ ޖޫން 18 ސެސެލް ޤ...`
 
 **Context Size 4:**
 
-1. `ެ ވ ެ . ހ ަ މ ަ އ ި ގ ަ ނ ް ނ ަ އ ި ޖ`
-2. `ގ ަ އ ި މ ި ވ ަ ނ ް ތ ަ ކ ެ ތ ީ ގ ަ އ`
-3. `އ ެ ވ ެ . ފ ަ ތ ަ ށ ް ޢ ަ މ ަ ލ ާ ގ ެ`
+1. `ބަންދު ދުވަސްތަކާއި ފާހަގަ ކުރެވޭ ދުވަހެއްގެ ގޮތުގައި ދިވެހިރާއްޖެ ޖުމުހޫރީ ދުވަސް`
+2. `ދުވަސްތަކާއި ފާހަގަ ކުރެވޭ ދުވަހެއްގެ ގޮތުގައި ޖުލައި 4 އެމެރިކާގެ މިނިވަން ދުވަސް ޖުލައި 4 ފިލިޕީނޯ...`
+3. `އުފަންވި މީހުން މަރުވި މީހުން ބަންދު ދުވަސްތަކާއި ފާހަގަ ކުރެވޭ ދުވަހެއްގެ ގޮތުގައި ކުޑަކުދިންގެ ދުވ...`
+
+
+### Generated Text Samples (Subword-based)
+
+Below are text samples generated from each subword-based Markov chain model:
+
+**Context Size 1:**
+
+1. `_ދެފައިން_އަދ._އަލް_ފައި_`
+2. `ން_ޒުވާ_ފައެވެ._މަރުނުވާ_e`
+3. `އި_ބޭބޭހެއުފެށިމަދުވަޑަކަލާގެ_`
+
+**Context Size 2:**
+
+1. `ން_•_pectight:_މިސްކި`
+2. `ގެ_ކުރައްވަމުން_ރުސް_ގޮމާ_ދިރު`
+3. `._މިން_ކަރައާއި_އޮތް_އިންޑަރު`
+
+**Context Size 3:**
+
+1. `ވެ._ކޯފުއްޕި_ޖެހުމުން_ބޭރުގައްޔާ`
+2. `ގައި_ޚިދުމަތްކުރައްވާފައެވެ._ވަނަ`
+3. `އެވެ._މިއީ_ފަރި_ރީކޯ_މޫސަބޭގެ_`
+
+**Context Size 4:**
+
+1. `އެވެ._ނާސްޕަތީ_ގައި_އަޅުގަނޑުމެން`
+2. `ކެވެ._އެއީ_ރޭގަނޑު_ގިރާކުރި_ތަޖް`
+3. `_އަދި_ހޯދިފައެއް_ނުލިބި_އެވެ._އު`
 
 
 ### Key Findings
 
-- **Best Predictability:** Context-2 with 70.1% predictability
+- **Best Predictability:** Context-4 (word) with 98.0% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (192,399 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (538,145 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -245,64 +346,64 @@ Below are text samples generated from each Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 6,434 |
-| Total Tokens | 3,460,301 |
-| Mean Frequency | 537.81 |
+| Vocabulary Size | 51,567 |
+| Total Tokens | 801,622 |
+| Mean Frequency | 15.55 |
 | Median Frequency | 3 |
-| Frequency Std Dev | 11173.10 |
+| Frequency Std Dev | 104.10 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | އ | 509,388 |
-| 2 | ނ | 367,732 |
-| 3 | މ | 247,599 |
-| 4 | ރ | 246,229 |
-| 5 | ވ | 242,170 |
-| 6 | ކ | 226,840 |
-| 7 | ގ | 210,999 |
-| 8 | ތ | 158,312 |
-| 9 | ދ | 142,011 |
-| 10 | ހ | 138,134 |
+| 1 | އަދި | 9,274 |
+| 2 | އެވެ | 6,692 |
+| 3 | އަކީ | 5,688 |
+| 4 | ވަނަ | 5,329 |
+| 5 | ނުވަތަ | 4,623 |
+| 6 | ވެސް | 4,608 |
+| 7 | އެންމެ | 4,606 |
+| 8 | ގެ | 3,870 |
+| 9 | މި | 3,411 |
+| 10 | އާއި | 3,404 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | علاء | 2 |
-| 2 | حاشية | 2 |
-| 3 | عابدين | 2 |
-| 4 | ٱق | 2 |
-| 5 | حصن | 2 |
-| 6 | 1972ވ | 2 |
-| 7 | abdul_raheem_abdulla_portrait | 2 |
-| 8 | 112x112pxޢ | 2 |
-| 9 | 282ވ | 2 |
-| 10 | costus | 2 |
+| 1 | ޤާނޫނެއްގައި | 2 |
+| 2 | ކަނޑައަޅައިފައިވާ | 2 |
+| 3 | އިސްތިއުނާފަށް | 2 |
+| 4 | ތަޢާރުޟުވާކަމަށް | 2 |
+| 5 | ޓްރައިބިއުނަލަކުން | 2 |
+| 6 | އެންޓަޓެއިންމަންޓުން | 2 |
+| 7 | costus | 2 |
+| 8 | ހުއިސުނަކީ | 2 |
+| 9 | fatah | 2 |
+| 10 | ސަބްސްކްރައިބް | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 1.2153 |
-| R² (Goodness of Fit) | 0.951559 |
+| Zipf Coefficient | 0.9604 |
+| R² (Goodness of Fit) | 0.990212 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 98.4% |
-| Top 1,000 | 99.4% |
-| Top 5,000 | 99.9% |
-| Top 10,000 | 0.0% |
+| Top 100 | 21.5% |
+| Top 1,000 | 48.5% |
+| Top 5,000 | 71.9% |
+| Top 10,000 | 81.3% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9516 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 98.4% of corpus
-- **Long Tail:** -3,566 words needed for remaining 100.0% coverage
+- **Zipf Compliance:** R²=0.9902 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 21.5% of corpus
+- **Long Tail:** 41,567 words needed for remaining 18.7% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -315,24 +416,123 @@ Below are text samples generated from each Markov chain model:
 
 ![t-SNE Sentences](visualizations/tsne_sentences.png)
 
-### Model Comparison
 
-| Model | Vocab Size | Dimension | Avg Norm | Std Norm | Isotropy |
-|-------|------------|-----------|----------|----------|----------|
-| **mono_32d** | 20,888 | 32 | 3.934 | 0.875 | 0.8870 🏆 |
-| **mono_64d** | 20,888 | 64 | 4.609 | 0.777 | 0.8593 |
-| **mono_128d** | 20,888 | 128 | 5.126 | 0.699 | 0.7135 |
-| **embeddings_enhanced** | 0 | 0 | 0.000 | 0.000 | 0.0000 |
+### 5.1 Cross-Lingual Alignment
+
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
+
+
+### 5.2 Model Comparison
+
+| Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
+|-------|-----------|----------|------------------|---------------|----------------|
+| **mono_32d** | 32 | 0.8795 | 0.3207 | N/A | N/A |
+| **mono_64d** | 64 | 0.8617 | 0.2441 | N/A | N/A |
+| **mono_128d** | 128 | 0.6946 | 0.1877 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8795 🏆 | 0.3125 | 0.0040 | 0.0580 |
+| **aligned_64d** | 64 | 0.8617 | 0.2426 | 0.0300 | 0.1720 |
+| **aligned_128d** | 128 | 0.6946 | 0.1963 | 0.0620 | 0.2160 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.8870 (more uniform distribution)
-- **Dimension Trade-off:** Higher dimensions capture more semantics but reduce isotropy
-- **Vocabulary Coverage:** All models cover 20,888 words
-- **Recommendation:** 100d for balanced semantic capture and efficiency
+- **Best Isotropy:** aligned_32d with 0.8795 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.2507. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 6.2% R@1 in cross-lingual retrieval.
+- **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
-## 6. Summary & Recommendations
+## 6.  Morphological Analysis (Experimental)
+
+This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
+
+### 6.1 Productivity & Complexity
+
+| Metric | Value | Interpretation | Recommendation |
+|--------|-------|----------------|----------------|
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **-0.063** | Low formulaic content | - |
+
+### 6.2 Affix Inventory (Productive Units)
+
+These are the most productive prefixes and suffixes identified by sampling the vocabulary for global substitutability patterns. A unit is considered an affix if stripping it leaves a valid stem that appears in other contexts.
+
+#### Productive Prefixes
+| Prefix | Examples |
+|--------|----------|
+| `-އެ` | އެންވައިރޮމަންޓަލް, އެތިސްޓުންނެވެ, އެދެބޭކަލުންގެ |
+| `-އަ` | އަމަލުތައް, އަބްދުއްރަހުމާނު, އަހައްމާއިދީ |
+| `-މަ` | މައްޗައް, މަދަވީ, މަސައްކަތްޕުޅާއި |
+| `-އި` | އިއްވި, އިނާމެކެވެ, އިނބަރަސްކަލާނގެ |
+| `-ބަ` | ބައްޕާފުޅެވެ, ބަށީގެ, ބަނޑުހައިވުން |
+| `-މި` | މިޔަރުތައް, މިޑުލާ, މިޗިގަންގެ |
+
+#### Productive Suffixes
+| Suffix | Examples |
+|--------|----------|
+| `-ް` | ރަނގަޅުކޮށް, ތައިރޮޑް, ރަދީފް |
+| `-ެ` | ބައްޕާފުޅެވެ, ޞޫފީންގެ, މުޅިރާއްޖޭގެ |
+| `-ި` | ގުޅިފައި, ކުރީކޮޅުގަޔާއި, ކާއިނާތުގައި |
+| `-ން` | ބަނޑުހައިވުން, ފޮނުވާލުމުން, ދޭކަން |
+| `-ގެ` | ޞޫފީންގެ, މުޅިރާއްޖޭގެ, ބަށީގެ |
+| `-އި` | ގުޅިފައި, ކުރީކޮޅުގަޔާއި, ކާއިނާތުގައި |
+| `-ވެ` | ބައްޕާފުޅެވެ, އުފަންކޮށްފައެވެ, ތިއޭޓަރެވެ |
+| `-ެވެ` | ބައްޕާފުޅެވެ, އުފަންކޮށްފައެވެ, ތިއޭޓަރެވެ |
+
+### 6.3 Bound Stems (Lexical Roots)
+
+Bound stems are high-frequency subword units that are semantically cohesive but rarely appear as standalone words. These often correspond to the 'core' of a word that requires inflection or derivation to be valid.
+
+*No significant bound stems detected.*
+
+
+### 6.4 Affix Compatibility (Co-occurrence)
+
+This table shows which prefixes and suffixes most frequently co-occur on the same stems, revealing the 'stacking' rules of the language's morphology.
+
+| Prefix | Suffix | Frequency | Examples |
+|--------|--------|-----------|----------|
+| `-އެ` | `-ް` | 155 words | އެއަކުން, އެކަކަށް |
+| `-މަ` | `-ް` | 107 words | މަސްތަކެއް, މަރާގުޅޭގޮތުން |
+| `-އަ` | `-ް` | 104 words | އަހަރުތަކަކަށް, އަލްއުސްތާޒް |
+| `-އަ` | `-ެ` | 102 words | އަންތަނަނާރިވޯއެވެ, އަކަށެވެ |
+| `-އި` | `-ް` | 91 words | އިތުރުވާން, އިއްޒަތްތެރިކަން |
+| `-އެ` | `-ެ` | 87 words | އެމެރިކާގައެވެ, އެއްޗެވެ |
+| `-މި` | `-ް` | 74 words | މިޞްރުން, މިޞްރަށް |
+| `-މަ` | `-ެ` | 71 words | މަދޫގެ, މަރުހަލާއެކެވެ |
+| `-ބަ` | `-ް` | 69 words | ބަހާއެއް, ބަދަލުކޮށްގެން |
+| `-ބަ` | `-ެ` | 61 words | ބަނޑޭރިގެއިންނެވެ, ބަދަރުންނެވެ |
+
+### 6.5 Recursive Morpheme Segmentation
+
+Using **Recursive Hierarchical Substitutability**, we decompose complex words into their constituent morphemes. This approach handles nested affixes (e.g., `prefix-prefix-root-suffix`).
+
+| Word | Suggested Split | Confidence | Stem |
+|------|-----------------|------------|------|
+| ދިމާވެގެން | **`ދިމާ-ވެ-ގެ-ން`** | 7.5 | `ދިމާ` |
+| ބުރައިގެން | **`ބުރަ-އި-ގެ-ން`** | 7.5 | `ބުރަ` |
+| މީހުންނާއިގެން | **`މީހުންނާ-އި-ގެ-ން`** | 7.5 | `މީހުންނާ` |
+| ބައްދަލުވެގެން | **`ބަ-އްދަލު-ވެ-ގެ-ން`** | 6.0 | `އްދަލު` |
+| އެއްކޮށްގެން | **`އެ-އްކޮ-ށް-ގެ-ން`** | 6.0 | `އްކޮ` |
+| އަނބުރައިގެން | **`އަ-ނބުރ-ައި-ގެ-ން`** | 6.0 | `ނބުރ` |
+| ގެއްލިގެން | **`ގެއްލި-ގެ-ން`** | 6.0 | `ގެއްލި` |
+| އެދަރިފުޅު | **`އެ-ދަރިފުޅު`** | 4.5 | `ދަރިފުޅު` |
+| ބްލޮކޭޑްގެ | **`ބްލޮކޭޑް-ގެ`** | 4.5 | `ބްލޮކޭޑް` |
+| ޤުރްއާނާއި | **`ޤުރްއާނާ-އި`** | 4.5 | `ޤުރްއާނާ` |
+| ޚިތާނުކޮށްގެން | **`ޚިތާނުކޮ-ށް-ގެ-ން`** | 4.5 | `ޚިތާނުކޮ` |
+| ވިސްނައިގެން | **`ވިސްނ-ައި-ގެ-ން`** | 4.5 | `ވިސްނ` |
+| މަޚްލޫޤުންގެ | **`މަ-ޚްލޫޤު-ން-ގެ`** | 4.5 | `ޚްލޫޤު` |
+| ކޮލަންބިޔާގެ | **`ކޮލަންބިޔާ-ގެ`** | 4.5 | `ކޮލަންބިޔާ` |
+| މައިގަނޑަކަށް | **`މަ-އި-ގަނޑަކަ-ށް`** | 4.5 | `ގަނޑަކަ` |
+
+### 6.6 Linguistic Interpretation
+
+> **Automated Insight:**
+The language Divehi shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
+
+---
+## 7. Summary & Recommendations
 
 ![Performance Dashboard](visualizations/performance_dashboard.png)
 
@@ -340,11 +540,12 @@ Below are text samples generated from each Markov chain model:
 
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
-| Tokenizer | **32k BPE** | Best compression (6.11x) with low UNK rate |
-| N-gram | **5-gram** | Lowest perplexity (280) |
-| Markov | **Context-4** | Highest predictability (70.1%) |
+| Tokenizer | **64k BPE** | Best compression (5.58x) |
+| N-gram | **2-gram** | Lowest perplexity (1,740) |
+| Markov | **Context-4** | Highest predictability (98.0%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
+
 ---
 ## Appendix: Metrics Glossary & Interpretation Guide
 
@@ -534,7 +735,8 @@ If you use these models in your research, please cite:
   author = {Kamali, Omar},
   title = {Wikilangs: Open NLP Models for Wikipedia Languages},
   year = {2025},
-  publisher = {HuggingFace},
+  doi = {10.5281/zenodo.18073153},
+  publisher = {Zenodo},
   url = {https://huggingface.co/wikilangs}
   institution = {Omneity Labs}
 }
@@ -550,7 +752,8 @@ MIT License - Free for academic and commercial use.
 - 🤗 Models: [huggingface.co/wikilangs](https://huggingface.co/wikilangs)
 - 📊 Data: [wikipedia-monthly](https://huggingface.co/datasets/omarkamali/wikipedia-monthly)
 - 👤 Author: [Omar Kamali](https://huggingface.co/omarkamali)
+- 🤝 Sponsor: [Featherless AI](https://featherless.ai)
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2025-12-30 08:42:33*
+*Report Date: 2026-01-04 02:56:36*

models/embeddings/aligned/dv_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1d482200b27a7aa72f7fc803a149cd0e1e1d1c25bd6b78e00d6d126d37be02b
+size 1045353050

models/embeddings/aligned/dv_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "dv", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/dv_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8fa99a544b6ec84b3887a5ae537884b9e651d21742505a171e47394508028c5
+size 65664

models/embeddings/aligned/dv_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "dv",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 1132,
+  "vocab_size": 20300
+}

models/embeddings/aligned/dv_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:594a1419ccffbac3d29e046ce8eaca474f411462e84ed39a457ecd7f140a8e05
+size 261762650

models/embeddings/aligned/dv_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "dv", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/dv_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba8d7cddfd89a1496933bef92bb6ee4363ef051141f0efea548dbf174a175b59
+size 4224

models/embeddings/aligned/dv_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "dv",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 1132,
+  "vocab_size": 20300
+}

models/embeddings/aligned/dv_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a467cbbe2282b7902ce905c3c43004730ef5f7a2b508bec086aacdcbcfb0ca55
+size 522959450

models/embeddings/aligned/dv_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "dv", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/dv_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d256e5a3e16132f82190dd9f2180269d7e3dc932331d58c7bf89cedf4e2f3b4
+size 16512

models/embeddings/aligned/dv_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "dv",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 1132,
+  "vocab_size": 20300
+}

models/embeddings/monolingual/dv_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17f06f2e5b33a30261c4b7b077e79cbd4c974eed9dd568506a068811d5b85523
-size 1045966622
+oid sha256:d1d482200b27a7aa72f7fc803a149cd0e1e1d1c25bd6b78e00d6d126d37be02b
+size 1045353050

models/embeddings/monolingual/dv_128d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 128,
   "version": "monolingual",
   "training_params": {
-    "dim": 128,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 128
   },
-  "vocab_size": 20888
+  "vocab_size": 20300
 }

models/embeddings/monolingual/dv_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:989d442fa29ab19c3397905afc409c5ea3fb2ac3f7ced4ecef5c5b59e58881c8
-size 261924638
+oid sha256:594a1419ccffbac3d29e046ce8eaca474f411462e84ed39a457ecd7f140a8e05
+size 261762650

models/embeddings/monolingual/dv_32d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 32,
   "version": "monolingual",
   "training_params": {
-    "dim": 32,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 32
   },
-  "vocab_size": 20888
+  "vocab_size": 20300
 }

models/embeddings/monolingual/dv_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f4c8f3716afafb553389e17fdcee49ce632da31ea6cd2b808bd64ec68e7bdece
-size 523271966
+oid sha256:a467cbbe2282b7902ce905c3c43004730ef5f7a2b508bec086aacdcbcfb0ca55
+size 522959450

models/embeddings/monolingual/dv_64d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 64,
   "version": "monolingual",
   "training_params": {
-    "dim": 64,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 64
   },
-  "vocab_size": 20888
+  "vocab_size": 20300
 }

models/subword_markov/dv_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b530227d1a204f72f8b65de1a305ae378f77c4e8b2924dfc23dbddc8dd38662
-size 81302
+oid sha256:b3307450866517bca770b8ff37162d57a876a6d14f5eafee3aa14b0a46482be2
+size 267538

models/subword_markov/dv_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "dv",
-  "unique_contexts": 1139,
-  "total_transitions": 8248184
+  "unique_contexts": 2104,
+  "total_transitions": 4608454
 }

models/subword_markov/dv_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e3eb962d2bf0387e41cfec8127109760ff53e27cb9bee987ddb74ecf1bca3092
-size 447616
+oid sha256:a8b21c8b0f713f90c4485be3e6a24dc1c0f68d648f081cf2cca0f751cbd80d57
+size 1420180

models/subword_markov/dv_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "dv",
-  "unique_contexts": 9934,
-  "total_transitions": 8243619
+  "unique_contexts": 38101,
+  "total_transitions": 4604081
 }

models/subword_markov/dv_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5988c0a1488f6ee94f379fe5d3c8e0f141b7cdd8004be19dd66020914eed032a
-size 1540276
+oid sha256:021ae456a8f45c05dc35ca22c3f068d9f4a571bb089760721df86bea334a49ff
+size 4845580

models/subword_markov/dv_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "dv",
-  "unique_contexts": 54858,
-  "total_transitions": 8239054
+  "unique_contexts": 187098,
+  "total_transitions": 4599708
 }

models/subword_markov/dv_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:385dd9376ce9fb1f3e0ab8e1dfb784f0bd42cb629f08765bdeacbdd9c859e5d7
-size 4185420
+oid sha256:2595e3f88e4d3ff88ab37af198725ec08c42c91cbb5e084a26bf6648e10b5907
+size 11840425

models/subword_markov/dv_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "dv",
-  "unique_contexts": 192399,
-  "total_transitions": 8234489
+  "unique_contexts": 538145,
+  "total_transitions": 4595335
 }

models/subword_ngram/dv_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc947db07c0da034539c406fe8a1f79e854caae8ba0e21db7a629f743943f4f4
-size 56113
+oid sha256:6649c852738e4dca60b7f09b5812cb6575f775f787771a2ea73faa04f07e9760
+size 243268

models/subword_ngram/dv_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "dv",
-  "unique_ngrams": 4183,
-  "total_ngrams": 8248184
+  "unique_ngrams": 17306,
+  "total_ngrams": 4608454
 }

models/subword_ngram/dv_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2de063fc8edb0676cdad6e6286041bbe5f0a9cc4a2d60744c57758a2f1b512be
-size 373741
+oid sha256:df7b26d1b6de16d1112f28a010e69200e2c0ce6f103032838409f56593466deb
+size 1204128

models/subword_ngram/dv_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "dv",
-  "unique_ngrams": 28100,
-  "total_ngrams": 8243619
+  "unique_ngrams": 83683,
+  "total_ngrams": 4604081
 }

models/subword_ngram/dv_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8a03c4da24aa6e3cbafac8d2e54359f6953d2a058be2d361e5f00d8b80e16540
-size 1401587
+oid sha256:a5c0fbfd42d7e89cc89ba25b9c4d8728ed0ab955caf2c9f43e8175c41a6f2aa5
+size 4016880

models/subword_ngram/dv_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "dv",
-  "unique_ngrams": 111102,
-  "total_ngrams": 8239054
+  "unique_ngrams": 264508,
+  "total_ngrams": 4599708
 }

models/subword_ngram/dv_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4dbc2c588c7c15676a1b44a5b49e6f4a783509de7b78755d365a11101ee245e
+size 6757063

models/subword_ngram/dv_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "dv",
+  "unique_ngrams": 409837,
+  "total_ngrams": 4595335
+}

models/tokenizer/dv_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b009d0f5075910344a5ec70654b5bb2f1453e75fdf1995842ff06f0a18b68e9
-size 613437
+oid sha256:d1c9eb7c4be528e9f39b3f1346888b36e643e65c0a594d6b6ed08c5ee650df2f
+size 613254

models/tokenizer/dv_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee60e395f3e281dda1c84ee287ea26df87d132e2cba1b17e96eb62b98922ba28
-size 1013278
+oid sha256:537aa90800cf8469cedd1c39978c264e9cf723ca276651647b0b88b09511d4c8
+size 1017550

models/tokenizer/dv_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da10dcfbcb21fe494de6a305e8ddfbe3a3b893d20e2f3ce5944c6d8f420440a5
-size 1826708
+oid sha256:15f182e34a936d64a09249529cf4fe6bd9d73e28fec863770c9e5da43d100863
+size 1888184

models/tokenizer/dv_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7360478c29b2bdf38819aa24a0e676d54c3133a34a63adb669179db4dcfaa256
-size 419002
+oid sha256:abd9d7c991e353b6cabc7362f2ea57937d0e4a80c23e7aa9fc3b342306f4c49c
+size 420093

models/vocabulary/dv_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:78a5a307e9810e9b095653562c638e91d2a71e0b2809564df0072f0c05d3e27b
-size 100935
+oid sha256:9b2b9d73aea17706d0f62012343fbbfcd863ec8856f63f71c43a55da5fc7bf73
+size 1042810

models/vocabulary/dv_vocabulary_metadata.json

@@ -1,16 +1,17 @@
 {
   "language": "dv",
-  "vocabulary_size": 6434,
+  "vocabulary_size": 51567,
+  "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.004362639272805053,
+    "type_token_ratio": 0.13912746062370807,
     "coverage": {
-      "top_100": 0.9816569669481213,
-      "top_1000": 0.9919847241323944,
-      "top_5000": 0.9966653223795554,
-      "top_10000": 0.9985200350187273
+      "top_100": 0.19776924109894373,
+      "top_1000": 0.4462941007009284,
+      "top_5000": 0.6616536952252623,
+      "top_10000": 0.7476577375862674
     },
-    "hapax_count": 8700,
-    "hapax_ratio": 0.5748645434121845,
-    "total_documents": 4565
+    "hapax_count": 69651,
+    "hapax_ratio": 0.5745928822452111,
+    "total_documents": 4373
   }
 }

models/word_markov/dv_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88a9fe6cb63c2277dd80fc088352c0625dfd2fa4ff7477a1b0f56bdee4ab18ce
-size 505042
+oid sha256:9292d565add8f3b46b5a81eb5592ad8584c8cfb75ba0ea0bb8cc7bd71589dd57
+size 8185665

models/word_markov/dv_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "dv",
-  "unique_contexts": 15381,
-  "total_transitions": 7006421
+  "unique_contexts": 120955,
+  "total_transitions": 866900
 }

models/word_markov/dv_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:655358b5e277faeeb357b22e4f035c2e76a94c1762518ad441fd4c2fdae93abb
-size 1283233
+oid sha256:3102b35e30e4b8fc34cbca97b08009b1cd7f6b99406e4a8ae1539150f5abcc08
+size 19496785

models/word_markov/dv_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "dv",
-  "unique_contexts": 58771,
-  "total_transitions": 7001856
+  "unique_contexts": 523452,
+  "total_transitions": 862527
 }