Upload all models and assets for diq (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: diq
-language_name: DIQ
+language_name: Dimli (individual language)
 language_family: iranian_other
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-iranian_other
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 3.820
+    value: 3.946
   - name: best_isotropy
     type: isotropy
-    value: 0.8131
+    value: 0.8232
   - name: vocabulary_size
     type: vocab
-    value: 103264
-generated: 2025-12-30
+    value: 0
+generated: 2026-01-04
 ---
 
-# DIQ - Wikilangs Models
+# Dimli (individual language) - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **DIQ** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Dimli (individual language)** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -44,12 +54,13 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 ### Models & Assets
 
 - Tokenizers (8k, 16k, 32k, 64k)
-- N-gram models (2, 3, 4-gram)
-- Markov chains (context of 1, 2, 3 and 4)
+- N-gram models (2, 3, 4, 5-gram)
+- Markov chains (context of 1, 2, 3, 4 and 5)
 - Subword N-gram and Markov chains
-- Embeddings in various sizes and dimensions
+- Embeddings in various sizes and dimensions (aligned and unaligned)
 - Language Vocabulary
 - Language Statistics
+
 ![Performance Dashboard](visualizations/performance_dashboard.png)
 
 ### Analysis and Evaluation
@@ -59,7 +70,8 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Summary & Recommendations](#6-summary--recommendations)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
+- [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
 
@@ -68,62 +80,57 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 ![Tokenizer Compression](visualizations/tokenizer_compression.png)
 
+![Tokenizer Fertility](visualizations/tokenizer_fertility.png)
+
+![Tokenizer OOV](visualizations/tokenizer_oov.png)
+
+![Total Tokens](visualizations/tokenizer_total_tokens.png)
+
 ### Results
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.034x | 2.99 | 0.0960% | 363,458 |
-| **16k** | 3.324x | 3.27 | 0.1052% | 331,721 |
-| **32k** | 3.590x | 3.53 | 0.1136% | 307,147 |
-| **64k** | 3.820x 🏆 | 3.76 | 0.1209% | 288,639 |
+| **8k** | 3.111x | 3.11 | 0.0973% | 324,747 |
+| **16k** | 3.420x | 3.42 | 0.1070% | 295,419 |
+| **32k** | 3.692x | 3.70 | 0.1155% | 273,644 |
+| **64k** | 3.946x 🏆 | 3.95 | 0.1234% | 256,028 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Exermont, dewleta Fransa de mıntıqaya Grand Esti de wılayetê Ardennesi de yew ko...`
+**Sample 1:** `.weir, nameyê bandıra sewiyaya serêna jeneriko (be İngılızki: Generic top-level ...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁ex er mont , ▁dewleta ▁fransa ▁de ▁mıntıqaya ▁grand ▁esti ... (+14 more)` | 24 |
-| 16k | `▁ex er mont , ▁dewleta ▁fransa ▁de ▁mıntıqaya ▁grand ▁esti ... (+14 more)` | 24 |
-| 32k | `▁ex er mont , ▁dewleta ▁fransa ▁de ▁mıntıqaya ▁grand ▁esti ... (+14 more)` | 24 |
-| 64k | `▁exermont , ▁dewleta ▁fransa ▁de ▁mıntıqaya ▁grand ▁esti ▁de ▁wılayetê ... (+12 more)` | 22 |
-
-**Sample 2:** `Seramey 
- Qewmıyayışê dınya 
-
- Qewmıyayışê memleketi 
-
- Biyayışi 
-
- Merdışi 
-
-
+| 8k | `▁. we ir , ▁nameyê ▁bandıra ▁sewiyaya ▁serêna ▁jeneriko ▁( ... (+19 more)` | 29 |
+| 16k | `▁. we ir , ▁nameyê ▁bandıra ▁sewiyaya ▁serêna ▁jeneriko ▁( ... (+19 more)` | 29 |
+| 32k | `▁. we ir , ▁nameyê ▁bandıra ▁sewiyaya ▁serêna ▁jeneriko ▁( ... (+19 more)` | 29 |
+| 64k | `▁. we ir , ▁nameyê ▁bandıra ▁sewiyaya ▁serêna ▁jeneriko ▁( ... (+19 more)` | 29 |
 
-...`
+**Sample 2:** `Bègues, dewleta Fransa de, mıntıqaya Auvergne-Rhône-Alpes miyan de yew komuna wı...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁seramey ▁qewmıyayışê ▁dınya ▁qewmıyayışê ▁memleketi ▁biyayışi ▁merdışi ▁kategoriye : 1 ... (+3 more)` | 13 |
-| 16k | `▁seramey ▁qewmıyayışê ▁dınya ▁qewmıyayışê ▁memleketi ▁biyayışi ▁merdışi ▁kategoriye : 1 ... (+3 more)` | 13 |
-| 32k | `▁seramey ▁qewmıyayışê ▁dınya ▁qewmıyayışê ▁memleketi ▁biyayışi ▁merdışi ▁kategoriye : 1 ... (+3 more)` | 13 |
-| 64k | `▁seramey ▁qewmıyayışê ▁dınya ▁qewmıyayışê ▁memleketi ▁biyayışi ▁merdışi ▁kategoriye : 1 ... (+3 more)` | 13 |
+| 8k | `▁b è gues , ▁dewleta ▁fransa ▁de , ▁mıntıqaya ▁auvergne ... (+15 more)` | 25 |
+| 16k | `▁b è gues , ▁dewleta ▁fransa ▁de , ▁mıntıqaya ▁auvergne ... (+15 more)` | 25 |
+| 32k | `▁b è gues , ▁dewleta ▁fransa ▁de , ▁mıntıqaya ▁auvergne ... (+15 more)` | 25 |
+| 64k | `▁bè gues , ▁dewleta ▁fransa ▁de , ▁mıntıqaya ▁auvergne - ... (+14 more)` | 24 |
 
-**Sample 3:** `Quincy-Basse  dewleta Fransa de mıntıqaya Hauts-de-France miyan de yew komunê qe...`
+**Sample 3:** `Cosne-d'Allier, dewleta Fransa de, mıntıqaya Overn-Ron-Alpan miyan de yew komuna...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁quin cy - bas se ▁dewleta ▁fransa ▁de ▁mıntıqaya ▁hauts ... (+19 more)` | 29 |
-| 16k | `▁quin cy - bas se ▁dewleta ▁fransa ▁de ▁mıntıqaya ▁hauts ... (+19 more)` | 29 |
-| 32k | `▁quincy - basse ▁dewleta ▁fransa ▁de ▁mıntıqaya ▁hauts - de ... (+17 more)` | 27 |
-| 64k | `▁quincy - basse ▁dewleta ▁fransa ▁de ▁mıntıqaya ▁hauts - de ... (+17 more)` | 27 |
+| 8k | `▁cos ne - d ' allier , ▁dewleta ▁fransa ▁de ... (+21 more)` | 31 |
+| 16k | `▁cos ne - d ' allier , ▁dewleta ▁fransa ▁de ... (+19 more)` | 29 |
+| 32k | `▁cos ne - d ' allier , ▁dewleta ▁fransa ▁de ... (+18 more)` | 28 |
+| 64k | `▁cos ne - d ' allier , ▁dewleta ▁fransa ▁de ... (+18 more)` | 28 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 3.820x compression
-- **Lowest UNK Rate:** 8k with 0.0960% unknown tokens
+- **Best Compression:** 64k achieves 3.946x compression
+- **Lowest UNK Rate:** 8k with 0.0973% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -132,57 +139,111 @@ Below are sample sentences tokenized with each vocabulary size:
 
 ![N-gram Perplexity](visualizations/ngram_perplexity.png)
 
+![N-gram Unique](visualizations/ngram_unique.png)
+
 ![N-gram Coverage](visualizations/ngram_coverage.png)
 
 ### Results
 
-| N-gram | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
-|--------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | 4,282 🏆 | 12.06 | 58,073 | 31.5% | 62.7% |
-| **2-gram** | 448 🏆 | 8.81 | 7,864 | 56.6% | 97.0% |
-| **3-gram** | 5,072 | 12.31 | 83,405 | 29.2% | 63.1% |
-| **3-gram** | 3,630 | 11.83 | 59,410 | 21.8% | 64.0% |
-| **4-gram** | 7,816 | 12.93 | 158,337 | 26.5% | 58.9% |
-| **4-gram** | 16,017 | 13.97 | 303,847 | 14.4% | 43.2% |
+| N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
+|--------|---------|------------|---------|----------------|------------------|-------------------|
+| **2-gram** | Word | 2,900 | 11.50 | 32,472 | 37.7% | 66.7% |
+| **2-gram** | Subword | 361 🏆 | 8.50 | 6,487 | 60.7% | 98.0% |
+| **3-gram** | Word | 2,363 | 11.21 | 37,780 | 38.7% | 72.4% |
+| **3-gram** | Subword | 3,111 | 11.60 | 45,197 | 22.3% | 67.0% |
+| **4-gram** | Word | 3,683 | 11.85 | 77,102 | 34.1% | 68.2% |
+| **4-gram** | Subword | 15,466 | 13.92 | 232,167 | 13.2% | 42.0% |
+| **5-gram** | Word | 3,179 | 11.63 | 61,892 | 33.7% | 70.0% |
+| **5-gram** | Subword | 42,786 | 15.38 | 597,917 | 10.1% | 34.5% |
 
 ### Top 5 N-grams by Size
 
-**2-grams:**
+**2-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `de ca` | 13,749 |
+| 2 | `de mıntıqaya` | 12,351 |
+| 3 | `ca gêno` | 11,945 |
+| 4 | `fransa de` | 11,892 |
+| 5 | `de yew` | 11,359 |
+
+**3-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `fransa de mıntıqaya` | 11,768 |
+| 2 | `dewleta fransa de` | 11,147 |
+| 3 | `de ca gêno` | 10,321 |
+| 4 | `bıvênên lista komunanê` | 8,041 |
+| 5 | `katalogê neweyê pêroyi` | 7,026 |
+
+**4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `kategoriye :` | 68,780 |
-| 2 | `disjunctive .` | 28,800 |
-| 3 | `& disjunctive` | 23,040 |
-| 4 | `i ̇` | 21,992 |
-| 5 | `, yew` | 18,635 |
+| 1 | `dewleta fransa de mıntıqaya` | 11,101 |
+| 2 | `katalogê neweyê pêroyi de` | 7,025 |
+| 3 | `cısım katalogê neweyê pêroyi` | 7,025 |
+| 4 | `no cısım katalogê neweyê` | 6,678 |
+| 5 | `lista cısmanê ngc gıreyê` | 6,644 |
 
-**3-grams:**
+**5-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `& disjunctive .` | 23,040 |
-| 2 | `: / /` | 15,940 |
-| 3 | `kategoriye : komunê` | 11,844 |
-| 4 | `ca gêno .` | 11,726 |
-| 5 | `dewleta fransa de` | 11,146 |
+| 1 | `cısım katalogê neweyê pêroyi de` | 7,024 |
+| 2 | `no cısım katalogê neweyê pêroyi` | 6,678 |
+| 3 | `lista cısmanê ngc gıreyê teberi` | 6,644 |
+| 4 | `de ca gêno de terefê` | 5,997 |
+| 5 | `asmêniyo no cısım katalogê neweyê` | 5,870 |
 
-**4-grams:**
+**2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `de ca gêno .` | 10,199 |
-| 2 | `https : / /` | 9,965 |
-| 3 | `. https : /` | 8,616 |
-| 4 | `cısım katalogê neweyê pêroyi` | 7,025 |
-| 5 | `katalogê neweyê pêroyi de` | 7,025 |
+| 1 | `a _` | 300,863 |
+| 2 | `e _` | 289,730 |
+| 3 | `a n` | 274,481 |
+| 4 | `ê _` | 267,322 |
+| 5 | `_ d` | 217,060 |
+
+**3-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `_ d e` | 157,628 |
+| 2 | `d e _` | 100,392 |
+| 3 | `o . _` | 73,592 |
+| 4 | `n ê _` | 68,515 |
+| 5 | `i y a` | 67,461 |
+
+**4-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `_ d e _` | 94,419 |
+| 2 | `a n ê _` | 43,769 |
+| 3 | `_ y e w` | 40,703 |
+| 4 | `_ k o m` | 40,690 |
+| 5 | `_ r a _` | 38,802 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `_ y e w _` | 36,954 |
+| 2 | `_ k o m u` | 34,451 |
+| 3 | `k o m u n` | 34,446 |
+| 4 | `_ b ı v ê` | 23,569 |
+| 5 | `b ı v ê n` | 23,557 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram with 448
+- **Best Perplexity:** 2-gram (subword) with 361
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~43% of corpus
+- **Coverage:** Top-1000 patterns cover ~35% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -190,55 +251,86 @@ Below are sample sentences tokenized with each vocabulary size:
 
 ![Markov Entropy](visualizations/markov_entropy.png)
 
+![Markov Contexts](visualizations/markov_contexts.png)
+
 ![Markov Branching](visualizations/markov_branching.png)
 
 ### Results
 
-| Context | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
-|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | 0.6352 | 1.553 | 4.26 | 261,554 | 36.5% |
-| **1** | 1.2157 | 2.323 | 8.73 | 2,560 | 0.0% |
-| **2** | 0.2227 | 1.167 | 1.59 | 1,111,370 | 77.7% |
-| **2** | 0.9540 | 1.937 | 5.91 | 22,347 | 4.6% |
-| **3** | 0.0834 | 1.059 | 1.17 | 1,762,135 | 91.7% |
-| **3** | 0.8172 | 1.762 | 4.15 | 131,963 | 18.3% |
-| **4** | 0.0378 🏆 | 1.027 | 1.08 | 2,058,989 | 96.2% |
-| **4** | 0.6869 🏆 | 1.610 | 2.93 | 547,289 | 31.3% |
+| Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
+|---------|---------|-------------|------------|------------------|-----------------|----------------|
+| **1** | Word | 0.7487 | 1.680 | 4.43 | 220,418 | 25.1% |
+| **1** | Subword | 0.9728 | 1.963 | 6.81 | 2,853 | 2.7% |
+| **2** | Word | 0.1773 | 1.131 | 1.38 | 970,777 | 82.3% |
+| **2** | Subword | 0.8745 | 1.833 | 5.24 | 19,403 | 12.6% |
+| **3** | Word | 0.0542 | 1.038 | 1.10 | 1,326,261 | 94.6% |
+| **3** | Subword | 0.7728 | 1.709 | 4.01 | 101,524 | 22.7% |
+| **4** | Word | 0.0216 🏆 | 1.015 | 1.04 | 1,442,368 | 97.8% |
+| **4** | Subword | 0.6913 | 1.615 | 2.98 | 406,622 | 30.9% |
 
-### Generated Text Samples
+### Generated Text Samples (Word-based)
 
-Below are text samples generated from each Markov chain model:
+Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `. rocvetışê afrika be directxo newe veti bi albi 82 77 , 245 . embıryani nıfus`
-2. `, elektrik û ziraet û tsunamiyê okyanusê tetisi de taxımestareyê balina miyan de michel de deyıranê`
-3. `- a crooked road • • behind 2004 ) ke vera cı kerdê porê xo elazığ`
+1. `de biyê ke yew belediyaya sûkê wayiye nıfus grafikê diagrami sero gorey serran ra nıfusê vilasantar`
+2. `ra nıfusê anouldi website resayış 14 807 windsor ontario kanada yew qezay lalapaşaya ekonomiye be ro...`
+3. `yew komunê aulnois beaufremont de anciyao embıryani nıfus bıvênên qam hewahebur kelek u nameyê bandı...`
 
 **Context Size 2:**
 
-1. `kategoriye : muzik kategoriye : meqaleyê ke be naman ra nuseno . weşanxaneyê roşna , dîyarbekir 2013`
-2. `disjunctive . libell_d_acheminement & disjunctive . code_commune_insee & disjunctive . nom_de_la_com...`
-3. `& disjunctive . code_postal & disjunctive . libell_d_acheminement & disjunctive . code_commune_insee...`
+1. `de ca gêno schleswig holsteini de wılayetê ardennesi de yew serra teqwimiya seramey biyayış gaius pl...`
+2. `de mıntıqaya normandiya de ca gêno xızmete gesnes en argonne ca gênê xızmete rozerotte de şebekey aw...`
+3. `ca gêno bıvênên lista komunanê loire atlantique pays de la loire de ca gêna xızmete escouloubre de`
 
 **Context Size 3:**
 
-1. `& disjunctive . ligne_5 xızmete essegney de şebekey awe esto û sistemê kanalizasyoni ratneyao . xeta...`
-2. `: / / aladin . u - strasbg . frno cısım katalogê neweyê pêroyi de ca gêno .`
-3. `kategoriye : komunê seine - et - moselley lista qezanê grand esti lista kantonanê grand esti çımey k...`
+1. `fransa de mıntıqaya occitanie de ca gêna xızmete trausse de şebekey awe esto û sistemê kanalizasyoni...`
+2. `dewleta fransa de mıntıqaya auvergne rhône alpesi miyan de yew komuna bıvênên lista komunanê seine e...`
+3. `de ca gêno embıryani nıfus grafikê diagrami sero gorey seran ra nıfusê sandiás bıvênên belediyey our...`
 
 **Context Size 4:**
 
-1. `de ca gêno . https : / / nisanyanmap . com / ? y = tepehan & t =`
-2. `https : / / datanova . laposte . fr / explore / dataset / laposte_hexasmal / information / ?`
-3. `. https : / / datanova . laposte . fr / explore / dataset / laposte_hexasmal / information /`
+1. `dewleta fransa de mıntıqaya grand esti de wılayetê vosgesi dero komuni 31 87 km2 ca gêno dormey herb...`
+2. `katalogê neweyê pêroyi de komê estareyanê miyan de ca gêno de terefê i ra keşıf biyo bıvênên asmên g...`
+3. `cısım katalogê neweyê pêroyi de komê estareyanê miyan de ca gêno de terefê astronom i ra keşıf biyo ...`
+
+
+### Generated Text Samples (Subword-based)
+
+Below are text samples generated from each subword-based Markov chain model:
+
+**Context Size 1:**
+
+1. `_6_gaus-seyan_zı`
+2. `eyirdê_ardullale`
+3. `anên_d_usi_n-cet`
+
+**Context Size 2:**
+
+1. `a_fra_hun_no_û_ho`
+2. `e_letektempar_–_d`
+3. `anê_man_lolynsall`
+
+**Context Size 3:**
+
+1. `_de_temê_ki_sec,_y`
+2. `de_verneyo_ra_nows`
+3. `o._telebebat_yılbı`
+
+**Context Size 4:**
+
+1. `_de_komunê_wılayetê`
+2. `anê_muzisyeno,_ber_`
+3. `_yew_film_rol_çakal`
 
 
 ### Key Findings
 
-- **Best Predictability:** Context-4 with 96.2% predictability
+- **Best Predictability:** Context-4 (word) with 97.8% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (547,289 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (406,622 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -254,33 +346,33 @@ Below are text samples generated from each Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 103,264 |
-| Total Tokens | 2,826,953 |
-| Mean Frequency | 27.38 |
+| Vocabulary Size | 92,779 |
+| Total Tokens | 2,332,304 |
+| Mean Frequency | 25.14 |
 | Median Frequency | 3 |
-| Frequency Std Dev | 563.36 |
+| Frequency Std Dev | 515.39 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | de | 116,092 |
-| 2 | kategoriye | 68,789 |
-| 3 | ra | 40,615 |
-| 4 | yew | 37,444 |
-| 5 | i | 30,825 |
-| 6 | disjunctive | 28,800 |
-| 7 | u | 27,646 |
-| 8 | bıvênên | 23,479 |
-| 9 | komunê | 22,120 |
-| 10 | û | 21,949 |
+| 1 | de | 115,037 |
+| 2 | ra | 40,569 |
+| 3 | yew | 37,084 |
+| 4 | u | 26,509 |
+| 5 | bıvênên | 23,466 |
+| 6 | û | 21,932 |
+| 7 | lista | 20,682 |
+| 8 | ca | 17,900 |
+| 9 | dewleta | 17,340 |
+| 10 | ke | 16,742 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | vizyonkewtış | 2 |
-| 2 | namehetkarkaykeritewrnoti | 2 |
+| 1 | aksiyongerilim | 2 |
+| 2 | vizyonkewtış | 2 |
 | 3 | sude | 2 |
 | 4 | alınca | 2 |
 | 5 | vurmaz | 2 |
@@ -294,24 +386,24 @@ Below are text samples generated from each Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 1.1065 |
-| R² (Goodness of Fit) | 0.997056 |
+| Zipf Coefficient | 1.0696 |
+| R² (Goodness of Fit) | 0.997357 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 39.3% |
-| Top 1,000 | 66.0% |
-| Top 5,000 | 79.3% |
-| Top 10,000 | 84.5% |
+| Top 100 | 39.8% |
+| Top 1,000 | 65.1% |
+| Top 5,000 | 78.5% |
+| Top 10,000 | 84.0% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9971 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 39.3% of corpus
-- **Long Tail:** 93,264 words needed for remaining 15.5% coverage
+- **Zipf Compliance:** R²=0.9974 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 39.8% of corpus
+- **Long Tail:** 82,779 words needed for remaining 16.0% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -324,24 +416,114 @@ Below are text samples generated from each Markov chain model:
 
 ![t-SNE Sentences](visualizations/tsne_sentences.png)
 
-### Model Comparison
 
-| Model | Vocab Size | Dimension | Avg Norm | Std Norm | Isotropy |
-|-------|------------|-----------|----------|----------|----------|
-| **mono_32d** | 38,727 | 32 | 4.943 | 1.253 | 0.8131 🏆 |
-| **mono_64d** | 38,727 | 64 | 5.521 | 1.070 | 0.7958 |
-| **mono_128d** | 38,727 | 128 | 6.001 | 0.853 | 0.6610 |
-| **embeddings_enhanced** | 0 | 0 | 0.000 | 0.000 | 0.0000 |
+### 5.1 Cross-Lingual Alignment
+
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
+
+
+### 5.2 Model Comparison
+
+| Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
+|-------|-----------|----------|------------------|---------------|----------------|
+| **mono_32d** | 32 | 0.8232 | 0.3686 | N/A | N/A |
+| **mono_64d** | 64 | 0.7882 | 0.3130 | N/A | N/A |
+| **mono_128d** | 128 | 0.5576 | 0.2631 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8232 🏆 | 0.3734 | 0.0360 | 0.2220 |
+| **aligned_64d** | 64 | 0.7882 | 0.3026 | 0.0680 | 0.3100 |
+| **aligned_128d** | 128 | 0.5576 | 0.2680 | 0.1060 | 0.4260 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.8131 (more uniform distribution)
-- **Dimension Trade-off:** Higher dimensions capture more semantics but reduce isotropy
-- **Vocabulary Coverage:** All models cover 38,727 words
-- **Recommendation:** 100d for balanced semantic capture and efficiency
+- **Best Isotropy:** aligned_32d with 0.8232 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.3148. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 10.6% R@1 in cross-lingual retrieval.
+- **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
-## 6. Summary & Recommendations
+## 6.  Morphological Analysis (Experimental)
+
+This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
+
+### 6.1 Productivity & Complexity
+
+| Metric | Value | Interpretation | Recommendation |
+|--------|-------|----------------|----------------|
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **1.030** | High formulaic/idiomatic content | - |
+
+### 6.2 Affix Inventory (Productive Units)
+
+These are the most productive prefixes and suffixes identified by sampling the vocabulary for global substitutability patterns. A unit is considered an affix if stripping it leaves a valid stem that appears in other contexts.
+
+#### Productive Prefixes
+| Prefix | Examples |
+|--------|----------|
+
+#### Productive Suffixes
+| Suffix | Examples |
+|--------|----------|
+| `-an` | ban, yewbiyayiyan, algan |
+
+### 6.3 Bound Stems (Lexical Roots)
+
+Bound stems are high-frequency subword units that are semantically cohesive but rarely appear as standalone words. These often correspond to the 'core' of a word that requires inflection or derivation to be valid.
+
+| Stem | Cohesion | Substitutability | Examples |
+|------|----------|------------------|----------|
+| `iyay` | 1.76x | 207 contexts | niyay, siyay, şiyay |
+| `iyan` | 1.73x | 143 contexts | biyan, niyan, ziyan |
+| `ista` | 1.71x | 64 contexts | kista, lista, vista |
+| `eber` | 1.92x | 37 contexts | teber, zeber, xeber |
+| `wlet` | 2.29x | 20 contexts | dewlet, dewletu, dewleto |
+| `ewle` | 2.23x | 20 contexts | dewle, sewle, hewle |
+| `leta` | 1.95x | 30 contexts | letan, aleta, ğeleta |
+| `nter` | 1.78x | 41 contexts | enter, inter, anter |
+| `rans` | 1.84x | 35 contexts | crans, frans, trans |
+| `laye` | 2.00x | 23 contexts | claye, layer, alaye |
+| `ıntı` | 2.38x | 12 contexts | alıntı, saçıntı, çalıntı |
+| `ntıq` | 1.93x | 18 contexts | mantıq, mentıq, mentıqi |
+
+### 6.4 Affix Compatibility (Co-occurrence)
+
+This table shows which prefixes and suffixes most frequently co-occur on the same stems, revealing the 'stacking' rules of the language's morphology.
+
+*No significant affix co-occurrences detected.*
+
+
+### 6.5 Recursive Morpheme Segmentation
+
+Using **Recursive Hierarchical Substitutability**, we decompose complex words into their constituent morphemes. This approach handles nested affixes (e.g., `prefix-prefix-root-suffix`).
+
+| Word | Suggested Split | Confidence | Stem |
+|------|-----------------|------------|------|
+| vınderdışan | **`vınderdış-an`** | 4.5 | `vınderdış` |
+| hıkumetan | **`hıkumet-an`** | 4.5 | `hıkumet` |
+| pêxamberan | **`pêxamber-an`** | 4.5 | `pêxamber` |
+| destnuşteyan | **`destnuştey-an`** | 4.5 | `destnuştey` |
+| sekuleran | **`sekuler-an`** | 4.5 | `sekuler` |
+| beynelmılelan | **`beynelmılel-an`** | 4.5 | `beynelmılel` |
+| karxaneyan | **`karxaney-an`** | 4.5 | `karxaney` |
+| meqaleyan | **`meqaley-an`** | 4.5 | `meqaley` |
+| qerebegan | **`qerebeg-an`** | 1.5 | `qerebeg` |
+| boğazlıyan | **`boğazlıy-an`** | 1.5 | `boğazlıy` |
+| çıldirtan | **`çıldirt-an`** | 1.5 | `çıldirt` |
+| meheliyan | **`meheliy-an`** | 1.5 | `meheliy` |
+| saskatchewan | **`saskatchew-an`** | 1.5 | `saskatchew` |
+| kalimantan | **`kalimant-an`** | 1.5 | `kalimant` |
+| gentleman | **`gentlem-an`** | 1.5 | `gentlem` |
+
+### 6.6 Linguistic Interpretation
+
+> **Automated Insight:**
+The language Dimli (individual language) shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
+
+> **Note on Idiomaticity:** The high Idiomaticity Gap suggests a large number of frequent multi-word expressions or formulaic sequences that are statistically distinct from their component parts.
+
+---
+## 7. Summary & Recommendations
 
 ![Performance Dashboard](visualizations/performance_dashboard.png)
 
@@ -349,11 +531,12 @@ Below are text samples generated from each Markov chain model:
 
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
-| Tokenizer | **32k BPE** | Best compression (3.82x) with low UNK rate |
-| N-gram | **5-gram** | Lowest perplexity (448) |
-| Markov | **Context-4** | Highest predictability (96.2%) |
+| Tokenizer | **64k BPE** | Best compression (3.95x) |
+| N-gram | **2-gram** | Lowest perplexity (361) |
+| Markov | **Context-4** | Highest predictability (97.8%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
+
 ---
 ## Appendix: Metrics Glossary & Interpretation Guide
 
@@ -543,7 +726,8 @@ If you use these models in your research, please cite:
   author = {Kamali, Omar},
   title = {Wikilangs: Open NLP Models for Wikipedia Languages},
   year = {2025},
-  publisher = {HuggingFace},
+  doi = {10.5281/zenodo.18073153},
+  publisher = {Zenodo},
   url = {https://huggingface.co/wikilangs}
   institution = {Omneity Labs}
 }
@@ -559,7 +743,8 @@ MIT License - Free for academic and commercial use.
 - 🤗 Models: [huggingface.co/wikilangs](https://huggingface.co/wikilangs)
 - 📊 Data: [wikipedia-monthly](https://huggingface.co/datasets/omarkamali/wikipedia-monthly)
 - 👤 Author: [Omar Kamali](https://huggingface.co/omarkamali)
+- 🤝 Sponsor: [Featherless AI](https://featherless.ai)
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2025-12-30 08:31:36*
+*Report Date: 2026-01-04 02:29:30*

models/embeddings/aligned/diq_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f4c06bf670ceb5b3f2fb9950b7c780ac8dca321facab94c6ffa452245a063e3
+size 1060747880

models/embeddings/aligned/diq_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "diq", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/diq_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbdde6fb2729128f90b97a3c72739e97d9a975b7bfc82ca51a36a412061dcd73
+size 65664

models/embeddings/aligned/diq_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "diq",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 13005,
+  "vocab_size": 35303
+}

models/embeddings/aligned/diq_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8df8da5c6f1446bcdb51c06943fd714e6c6ebe0c6265a1221197009d9a73bbf
+size 265635176

models/embeddings/aligned/diq_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "diq", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/diq_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21c41d20f27c902b8fab4d72e46d26292ea326ec56a34b288fdae7b7bb3f77e5
+size 4224

models/embeddings/aligned/diq_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "diq",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 13005,
+  "vocab_size": 35303
+}

models/embeddings/aligned/diq_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1ec35a19fb448cf17fbf358d7a83ea2b4608c5fd8248b04b7051e0183249c13
+size 530672744

models/embeddings/aligned/diq_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "diq", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/diq_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3bf851603178a9372a996a6bf3daea7479f7d418b3e3916af288fe72685c1b8
+size 16512

models/embeddings/aligned/diq_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "diq",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 13005,
+  "vocab_size": 35303
+}

models/embeddings/monolingual/diq_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:804523ae2f81ee8e224553b5a307a5fb9ee71479d0b0883658ac8639a49d1b1e
-size 1064321464
+oid sha256:5f4c06bf670ceb5b3f2fb9950b7c780ac8dca321facab94c6ffa452245a063e3
+size 1060747880

models/embeddings/monolingual/diq_128d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 128,
   "version": "monolingual",
   "training_params": {
-    "dim": 128,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 128
   },
-  "vocab_size": 38727
+  "vocab_size": 35303
 }

models/embeddings/monolingual/diq_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c98d6962981950297f6fa20ed2b955b6d11dafcbc20a5defeba0e6055feef87
-size 266579128
+oid sha256:d8df8da5c6f1446bcdb51c06943fd714e6c6ebe0c6265a1221197009d9a73bbf
+size 265635176

models/embeddings/monolingual/diq_32d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 32,
   "version": "monolingual",
   "training_params": {
-    "dim": 32,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 32
   },
-  "vocab_size": 38727
+  "vocab_size": 35303
 }

models/embeddings/monolingual/diq_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f22f12464b073e65e566e7457eaaf6328b8cb9fe2a5358b39e18e74b7665a73b
-size 532493240
+oid sha256:e1ec35a19fb448cf17fbf358d7a83ea2b4608c5fd8248b04b7051e0183249c13
+size 530672744

models/embeddings/monolingual/diq_64d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 64,
   "version": "monolingual",
   "training_params": {
-    "dim": 64,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 64
   },
-  "vocab_size": 38727
+  "vocab_size": 35303
 }

models/subword_markov/diq_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c228cfcd048e4029165c928626e754b73d0208d6b5c3375e51f9c986edde5eaa
-size 166055
+oid sha256:362bfda449239bbc8100af130e81ecdbe01b2e800c40125da584537d4c980702
+size 151403

models/subword_markov/diq_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "diq",
-  "unique_contexts": 2560,
-  "total_transitions": 20497513
+  "unique_contexts": 2853,
+  "total_transitions": 15467977
 }

models/subword_markov/diq_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a9711d19dc369ab84d332b058220fde2c686f635ed23c3b9ca77a79317878780
-size 1013394
+oid sha256:9ecb6abd7f57584c1c28720645c91b530fcd18344b75621afe169ac056820f1f
+size 819379

models/subword_markov/diq_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "diq",
-  "unique_contexts": 22347,
-  "total_transitions": 20455039
+  "unique_contexts": 19403,
+  "total_transitions": 15426449
 }

models/subword_markov/diq_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0dfbcc947a0f9cd16ecd7c242c39492b588588909988424ffbff76de677d84f4
-size 3980621
+oid sha256:cc03d4f755cf38c017837bb6b777f47a98fb88f2214afec6d4c420bd2caa6e6d
+size 2994129

models/subword_markov/diq_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "diq",
-  "unique_contexts": 131963,
-  "total_transitions": 20412565
+  "unique_contexts": 101524,
+  "total_transitions": 15384921
 }

models/subword_markov/diq_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad7282ca12a2e9edbedfa108834ec799debd726503f409b3d557251ccb7b408b
-size 11943049
+oid sha256:60bb83a4162c6a621753c787969df98551141c236b01f6da2e5b12378803c3fd
+size 9070939

models/subword_markov/diq_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "diq",
-  "unique_contexts": 547289,
-  "total_transitions": 20370091
+  "unique_contexts": 406622,
+  "total_transitions": 15343393
 }

models/subword_ngram/diq_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2a320ca415094d1b91be3ca94297478ea100000f9ac86e6b3da91b31a5ce6996
-size 102080
+oid sha256:688e3a7d7b3f14c65570b84420d7e744b86cb3184ea8f4d4c7b24620058bac2c
+size 85488

models/subword_ngram/diq_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "diq",
-  "unique_ngrams": 7864,
-  "total_ngrams": 20497513
+  "unique_ngrams": 6487,
+  "total_ngrams": 15467977
 }

models/subword_ngram/diq_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec4302da7328377c6001028acd1df7b6e98719386dcf762c5aed37c02568964b
-size 727256
+oid sha256:d27ccb9b0b3671c63db9ce363451669923dc7107d2348a396d65c420c9c72f71
+size 582747

models/subword_ngram/diq_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "diq",
-  "unique_ngrams": 59410,
-  "total_ngrams": 20455039
+  "unique_ngrams": 45197,
+  "total_ngrams": 15426449
 }

models/subword_ngram/diq_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b7cc2aed3d40bc42c4e6d130e4872f4d69762137ef4a06c5dc078af5d9c33630
-size 3386545
+oid sha256:277c06bfdb2b94a800f61d593b99b8308f473a6808237d0d32a4d000ffbf177b
+size 2615375

models/subword_ngram/diq_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "diq",
-  "unique_ngrams": 303847,
-  "total_ngrams": 20412565
+  "unique_ngrams": 232167,
+  "total_ngrams": 15384921
 }

models/subword_ngram/diq_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcb17e1957d58d268d8e5e255187286d591f312bdeabbb5df1f9a1c42500aede
+size 6944724

models/subword_ngram/diq_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "diq",
+  "unique_ngrams": 597917,
+  "total_ngrams": 15343393
+}

models/tokenizer/diq_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:682053db83da7a2a9475b2193c3362fa3f6f0fedd23d7a017efbb2df18dd8166
-size 498709
+oid sha256:c6aeb66620f8744cf0e32784c47b35389176e12f1c0f4fdbf2825d520667a0c2
+size 501892

models/tokenizer/diq_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:24382f7df4d599f80252487348a45602a9d7f40b930353ca09e356a4d5c302df
-size 766632
+oid sha256:35510dc9bf136aa4abb0ddda35880c0a69cceca84811a923ce59c24dc3651560
+size 774471

models/tokenizer/diq_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2a8e420b9039b6d0191841d64e2122315b43646d1ef9342f8748ac0f05ab605f
-size 1320507
+oid sha256:e0aaeba9a6897644b7dd1c16332189187adf9d6e4dd3906c0cc99776e79dada6
+size 1325280

models/tokenizer/diq_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe1d5016b848a4f664edfcca0f8bf93aac8a30b6accbf840d63d8f6a62a5ef25
-size 368537
+oid sha256:2b427813c7f6131d8560f5fd7f17c2eb3829e4d194d0e1e81017fb7d56effe06
+size 370411

models/vocabulary/diq_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad9cfb0e5369eee18a2d405a354f029fd075abcb43ab12c0615245cb8f1f51f6
-size 1737543
+oid sha256:5241548c40a5c49f67fab83621b14f04815636ff485772b815005f04df2aa01d
+size 1599436

models/vocabulary/diq_vocabulary_metadata.json

@@ -1,16 +1,17 @@
 {
   "language": "diq",
-  "vocabulary_size": 103264,
+  "vocabulary_size": 92779,
+  "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.0875893629715288,
+    "type_token_ratio": 0.08977346809710926,
     "coverage": {
-      "top_100": 0.3717204823732889,
-      "top_1000": 0.6248964459825731,
-      "top_5000": 0.7510549696381128,
-      "top_10000": 0.8001827039582843
+      "top_100": 0.37757051281009135,
+      "top_1000": 0.6169629052789705,
+      "top_5000": 0.7438497905222069,
+      "top_10000": 0.7963553952928055
     },
-    "hapax_count": 158204,
-    "hapax_ratio": 0.6050606575183196,
-    "total_documents": 42474
+    "hapax_count": 128100,
+    "hapax_ratio": 0.5799555412692017,
+    "total_documents": 41528
   }
 }

models/word_markov/diq_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e3872be140993f83d6a28dff1e81c7891ecc1623816a79a2057496cfe593a743
-size 10726510
+oid sha256:4bc0cf1f72396e6b00c96fe78f83848b8b1d235fdd850bd47e53e127098cdd6a
+size 9430828

models/word_markov/diq_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "diq",
-  "unique_contexts": 261554,
-  "total_transitions": 3986092
+  "unique_contexts": 220418,
+  "total_transitions": 2418876
 }

models/word_markov/diq_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:406a875d026390ecd4d7982509e4f3f152a51709e826dd04a3ee46ce7142b42f
-size 21768144
+oid sha256:e2fdd433ee82dc4cd355a465aa878b2be2f3b56c90432dcd035f55e52a884371
+size 18771699

models/word_markov/diq_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "diq",
-  "unique_contexts": 1111370,
-  "total_transitions": 3943618
+  "unique_contexts": 970777,
+  "total_transitions": 2377348
 }