Upload all models and assets for cv (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: cv
-language_name: CV
+language_name: Chuvash
 language_family: turkic_other
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-turkic_other
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 3.792
+    value: 3.778
   - name: best_isotropy
     type: isotropy
-    value: 0.8332
+    value: 0.8326
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# CV - Wikilangs Models
+# Chuvash - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **CV** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Chuvash** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.084x | 3.09 | 0.2455% | 244,836 |
-| **16k** | 3.356x | 3.36 | 0.2672% | 224,964 |
-| **32k** | 3.590x | 3.60 | 0.2857% | 210,324 |
-| **64k** | 3.792x 🏆 | 3.80 | 0.3018% | 199,115 |
+| **8k** | 3.075x | 3.08 | 0.2413% | 246,622 |
+| **16k** | 3.345x | 3.35 | 0.2625% | 226,699 |
+| **32k** | 3.576x | 3.58 | 0.2806% | 212,069 |
+| **64k** | 3.778x 🏆 | 3.78 | 0.2964% | 200,734 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `— Ăна хуплашкасене, тата пекех япаласене каясран сыхлама усă тата шалти Ĕç тата ...`
+**Sample 1:** `Вики: Вики Wiki Wiki WIKI (FM) Wiki wiki dollar Wiki Wiki Shuttle WikiWikiWeb Ви...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁— ▁ă на ▁хуп лаш ка сене , ▁тата ▁пекех ... (+24 more)` | 34 |
-| 16k | `▁— ▁ă на ▁хуп лаш ка сене , ▁тата ▁пекех ... (+24 more)` | 34 |
-| 32k | `▁— ▁ă на ▁хуп лашка сене , ▁тата ▁пекех ▁япаласене ... (+21 more)` | 31 |
-| 64k | `▁— ▁ă на ▁хуп лашка сене , ▁тата ▁пекех ▁япаласене ... (+20 more)` | 30 |
+| 8k | `▁вики : ▁вики ▁wik i ▁wik i ▁wik i ▁( ... (+41 more)` | 51 |
+| 16k | `▁вики : ▁вики ▁wiki ▁wiki ▁wiki ▁( f m ) ... (+28 more)` | 38 |
+| 32k | `▁вики : ▁вики ▁wiki ▁wiki ▁wiki ▁( fm ) ▁wiki ... (+25 more)` | 35 |
+| 64k | `▁вики : ▁вики ▁wiki ▁wiki ▁wiki ▁( fm ) ▁wiki ... (+23 more)` | 33 |
 
-**Sample 2:** `Шатдорф — () — коммуна, Швейцарири варринче Ури кантонне çын код — сайт хулисем ...`
+**Sample 2:** `Хро́мпик — ят е мар ят. Хромпик — калий Топоним Хромпик — çул Первоуральск (стан...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁ш ат дорф ▁— ▁() ▁— ▁коммуна , ▁швейцарири ▁варринче ... (+9 more)` | 19 |
-| 16k | `▁шат дорф ▁— ▁() ▁— ▁коммуна , ▁швейцарири ▁варринче ▁ури ... (+8 more)` | 18 |
-| 32k | `▁шат дорф ▁— ▁() ▁— ▁коммуна , ▁швейцарири ▁варринче ▁ури ... (+8 more)` | 18 |
-| 64k | `▁шат дорф ▁— ▁() ▁— ▁коммуна , ▁швейцарири ▁варринче ▁ури ... (+8 more)` | 18 |
+| 8k | `▁х ро ́м п ик ▁— ▁ят ▁е ▁мар ▁ят ... (+51 more)` | 61 |
+| 16k | `▁х ро ́м п ик ▁— ▁ят ▁е ▁мар ▁ят ... (+43 more)` | 53 |
+| 32k | `▁х ро ́м пик ▁— ▁ят ▁е ▁мар ▁ят . ... (+36 more)` | 46 |
+| 64k | `▁х ро ́м пик ▁— ▁ят ▁е ▁мар ▁ят . ... (+32 more)` | 42 |
 
-**Sample 3:** `Çĕр йышшисем – пуклак Малти урисем кайри урисем ытларах çĕр иртет. тата пурӑнать...`
+**Sample 3:** `Мушар — Республикин Куславкка ял. ял Коричев АССР Халах Вуламалли алфавитпа`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁çĕ р ▁йышшисем ▁– ▁пу к лак ▁малти ▁у рисем ... (+30 more)` | 40 |
-| 16k | `▁çĕ р ▁йышшисем ▁– ▁пу к лак ▁малти ▁урисем ▁кайри ... (+27 more)` | 37 |
-| 32k | `▁çĕ р ▁йышшисем ▁– ▁пук лак ▁малти ▁урисем ▁кайри ▁урисем ... (+26 more)` | 36 |
-| 64k | `▁çĕ р ▁йышшисем ▁– ▁пуклак ▁малти ▁урисем ▁кайри ▁урисем ▁ытларах ... (+25 more)` | 35 |
+| 8k | `▁му шар ▁— ▁республикин ▁куславкка ▁ял . ▁ял ▁кори чев ... (+4 more)` | 14 |
+| 16k | `▁му шар ▁— ▁республикин ▁куславкка ▁ял . ▁ял ▁кори чев ... (+4 more)` | 14 |
+| 32k | `▁му шар ▁— ▁республикин ▁куславкка ▁ял . ▁ял ▁коричев ▁асср ... (+3 more)` | 13 |
+| 64k | `▁му шар ▁— ▁республикин ▁куславкка ▁ял . ▁ял ▁коричев ▁асср ... (+3 more)` | 13 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 3.792x compression
-- **Lowest UNK Rate:** 8k with 0.2455% unknown tokens
+- **Best Compression:** 64k achieves 3.778x compression
+- **Lowest UNK Rate:** 8k with 0.2413% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 9,368 | 13.19 | 70,721 | 26.8% | 48.1% |
-| **2-gram** | Subword | 533 🏆 | 9.06 | 7,915 | 52.7% | 95.2% |
-| **3-gram** | Word | 8,221 | 13.01 | 88,916 | 30.4% | 52.4% |
-| **3-gram** | Subword | 4,932 | 12.27 | 69,443 | 17.2% | 56.3% |
-| **4-gram** | Word | 14,425 | 13.82 | 168,587 | 26.5% | 47.7% |
-| **4-gram** | Subword | 26,358 | 14.69 | 379,120 | 10.1% | 32.1% |
+| **2-gram** | Word | 9,473 | 13.21 | 71,211 | 26.6% | 47.9% |
+| **2-gram** | Subword | 532 🏆 | 9.06 | 7,908 | 52.7% | 95.2% |
+| **3-gram** | Word | 8,325 | 13.02 | 89,585 | 30.3% | 52.2% |
+| **3-gram** | Subword | 4,929 | 12.27 | 69,351 | 17.2% | 56.3% |
+| **4-gram** | Word | 14,593 | 13.83 | 169,630 | 26.4% | 47.5% |
+| **4-gram** | Subword | 26,364 | 14.69 | 378,926 | 10.1% | 32.1% |
+| **5-gram** | Word | 12,306 | 13.59 | 144,170 | 27.1% | 49.1% |
+| **5-gram** | Subword | 81,182 | 16.31 | 1,007,721 | 7.9% | 24.5% |
 
 ### Top 5 N-grams by Size
 
@@ -150,7 +162,7 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `шыв шыв` | 22,909 |
+| 1 | `шыв шыв` | 22,911 |
 | 2 | `территоринчи юханшыв` | 14,353 |
 | 3 | `территорипе юхать` | 13,579 |
 | 4 | `юхса юханшыв` | 13,517 |
@@ -162,56 +174,76 @@ Below are sample sentences tokenized with each vocabulary size:
 |------|--------|-------|
 | 1 | `рф экологи министерстви` | 11,700 |
 | 2 | `территорин шыв геоинформаци` | 11,389 |
-| 3 | `агентстви рф территорин` | 11,389 |
-| 4 | `рф территорин шыв` | 11,389 |
-| 5 | `федераци агентстви рф` | 11,389 |
+| 3 | `геоинформаци системин шыв` | 11,389 |
+| 4 | `федераци агентстви рф` | 11,389 |
+| 5 | `шыв федераци агентстви` | 11,389 |
 
 **4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `шыв геоинформаци системин шыв` | 11,389 |
-| 2 | `федераци агентстви рф территорин` | 11,389 |
+| 1 | `геоинформаци системин шыв шыв` | 11,389 |
+| 2 | `рф территорин шыв геоинформаци` | 11,389 |
 | 3 | `агентстви рф территорин шыв` | 11,389 |
-| 4 | `территорин шыв геоинформаци системин` | 11,389 |
-| 5 | `рф территорин шыв геоинформаци` | 11,389 |
+| 4 | `федераци агентстви рф территорин` | 11,389 |
+| 5 | `территорин шыв геоинформаци сис��емин` | 11,389 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `агентстви рф территорин шыв геоинформаци` | 11,389 |
+| 2 | `федераци агентстви рф территорин шыв` | 11,389 |
+| 3 | `шыв геоинформаци системин шыв шыв` | 11,389 |
+| 4 | `территорин шыв геоинформаци системин шыв` | 11,389 |
+| 5 | `шыв федераци агентстви рф территорин` | 11,389 |
 
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `. _` | 462,300 |
-| 2 | `а _` | 400,655 |
-| 3 | `и _` | 362,557 |
-| 4 | `— _` | 343,684 |
-| 5 | `_ —` | 341,328 |
+| 1 | `. _` | 465,426 |
+| 2 | `а _` | 402,164 |
+| 3 | `и _` | 363,006 |
+| 4 | `— _` | 346,175 |
+| 5 | `_ —` | 343,660 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `_ — _` | 340,403 |
-| 2 | `ш ы в` | 149,607 |
-| 3 | `ы в _` | 121,960 |
-| 4 | `_ ю х` | 94,722 |
-| 5 | `т е р` | 86,265 |
+| 1 | `_ — _` | 342,728 |
+| 2 | `ш ы в` | 149,577 |
+| 3 | `ы в _` | 121,922 |
+| 4 | `_ ю х` | 94,718 |
+| 5 | `т е р` | 86,508 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `ш ы в _` | 121,866 |
-| 2 | `_ ш ы в` | 85,504 |
-| 3 | `_ ю х а` | 76,923 |
-| 4 | `ю х а н` | 63,389 |
-| 5 | `х а н ш` | 63,293 |
+| 1 | `ш ы в _` | 121,828 |
+| 2 | `_ ш ы в` | 85,484 |
+| 3 | `_ ю х а` | 76,914 |
+| 4 | `ю х а н` | 63,379 |
+| 5 | `х а н ш` | 63,281 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `_ ш ы в _` | 83,923 |
+| 2 | `ю х а н ш` | 63,268 |
+| 3 | `х а н ш ы` | 63,265 |
+| 4 | `а н ш ы в` | 63,263 |
+| 5 | `_ ю х а н` | 62,475 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram (subword) with 533
+- **Best Perplexity:** 2-gram (subword) with 532
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~32% of corpus
+- **Coverage:** Top-1000 patterns cover ~25% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.7797 | 1.717 | 5.33 | 352,008 | 22.0% |
-| **1** | Subword | 0.6180 | 1.535 | 6.05 | 3,627 | 38.2% |
-| **2** | Word | 0.1825 | 1.135 | 1.40 | 1,864,001 | 81.8% |
-| **2** | Subword | 0.9070 | 1.875 | 6.21 | 21,896 | 9.3% |
-| **3** | Word | 0.0523 | 1.037 | 1.09 | 2,580,848 | 94.8% |
-| **3** | Subword | 0.8725 | 1.831 | 4.70 | 135,774 | 12.7% |
-| **4** | Word | 0.0222 🏆 | 1.016 | 1.04 | 2,780,564 | 97.8% |
-| **4** | Subword | 0.7090 | 1.635 | 3.14 | 637,656 | 29.1% |
+| **1** | Word | 0.7800 | 1.717 | 5.34 | 352,836 | 22.0% |
+| **1** | Subword | 0.6157 | 1.532 | 6.03 | 3,635 | 38.4% |
+| **2** | Word | 0.1829 | 1.135 | 1.40 | 1,869,675 | 81.7% |
+| **2** | Subword | 0.9040 | 1.871 | 6.19 | 21,903 | 9.6% |
+| **3** | Word | 0.0525 | 1.037 | 1.09 | 2,591,084 | 94.7% |
+| **3** | Subword | 0.8721 | 1.830 | 4.70 | 135,543 | 12.8% |
+| **4** | Word | 0.0223 🏆 | 1.016 | 1.04 | 2,792,400 | 97.8% |
+| **4** | Subword | 0.7095 | 1.635 | 3.14 | 636,890 | 29.1% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `шыв бассейн км2 экономика культура ял акимов анатолий тимофеевич разин остин пауэрс голдмембер найдж...`
-2. `юханшыв хура kara online куликовский п в сталин рузвельт черчилль в в карьере сампраса на экраны`
-3. `в о герард доу джонс в иван карабиц валерий валерьевич милици ричард iii вăл 15 pp`
+1. `шыв гидрологи бассейн шыв шыв геоинформаци системин шыв федераци агентстви рф территорин шыв геоинфо...`
+2. `юханшыв двина печора шыв федераци агентстви рф экологи министерстви республикин ао коми республики т...`
+3. `в цене чем предпочитают вспоминать и дефекты зрения м советская энциклопедия в унисон с любашей леро...`
 
 **Context Size 2:**
 
-1. `шыв шыв иртыш юханшыв бассейн шыв кара таз чиккинчен енисей чиккичен бассейн çук юханшыв кара обь ху...`
-2. `территоринчи юханшыв республики ростов ставрополь ен территорипе юхать юханшыва юхса юханшыв 33 км ю...`
-3. `территорипе юхать лелен еган юханшыва юхса юханшыв 7 500 км ытла b 15 калса çу çур çу`
+1. `шыв шыв тури обь иртыш шыв федераци агентстви рф территорин шыв геоинформаци системин шыв шыв тури о...`
+2. `территоринчи юханшыв рейн вестфали территорипе юхать юханшыв негус ях сулахай 13 км шыв шыв тури бас...`
+3. `территорипе юхать юханшыв мăн салым сулахай 220 км юхса юханшыв 12 км шыв шыв гидрологи бассейн том`
 
 **Context Size 3:**
 
-1. `федераци агентстви рф территорин шыв геоинформаци системин шыв шыв гидрологи гт бассейн том гт 08 гт...`
-2. `шыв шыв гидрологи гт бассейн том гт 11 гт 1 рф экологи министерстви республикин ао юпписем`
-3. `шыв федераци агентстви рф территорин шыв геоинформаци системин шыв шыв гидрологи бассейн том гт 15 г...`
+1. `федераци агентстви рф территорин шыв геоинформаци системин шыв шыв гидрологи гт бассейн том гт 15 гт...`
+2. `шыв федераци агентстви рф территорин шыв геоинформаци системин шыв шыв гидрологи бассейн том 15 3 рф...`
+3. `шыв геоинформаци системин шыв шыв гидрологи гт бассейн том гт 11 гт 1 рф экологи министерстви респуб...`
 
 **Context Size 4:**
 
-1. `шыв геоинформаци системин шыв шыв гидрологи бассейн том гт 15 гт 3 рф экологи министерстви автономи ...`
-2. `рф территорин шыв геоинформаци системин шыв шыв гидрологи бассейн том 15 3 рф экологи министерстви а...`
-3. `федераци агентстви рф территорин шыв геоинформаци системин шыв шыв гидрологи гт бассейн том гт 03 гт...`
+1. `шыв геоинформаци системин шыв шыв гидрологи гт бассейн том гт 03 гт 0 рф экологи министерстви ао рес...`
+2. `территорин шыв геоинформаци системин шыв шыв гидрологи бассейн том 15 3 рф экологи министерстви авто...`
+3. `геоинформаци системин шыв шыв гидрологи гт бассейн том гт 03 гт 0 рф экологи министерстви ао республ...`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `_kruk=_neesmblot`
-2. `адов_ин_пчнфи_ое`
-3. `изра_,_ацин_—_je`
+1. `_—_фикулигинци_в`
+2. `а,_;_улслаки_пид`
+3. `и_каспалименияни`
 
 **Context Size 2:**
 
-1. `._кий,_фрах_(3_г.`
-2. `а_—_перрипе_влеор`
-3. `и_э.в._афизм_—_де`
+1. `._—_торф_тыслана_`
+2. `а_медилостви_тута`
+3. `и_йышши_баллина_з`
 
 **Context Size 3:**
 
-1. `_—_дев_анчах_тата_`
-2. `шыва_сайтра_нингсе`
-3. `ыв_5_-6_км._шыв_чу`
+1. `_—_теминисем_астар`
+2. `шыв_—_мар_монтовол`
+3. `ыв_шыв._команицы:_`
 
 **Context Size 4:**
 
-1. `шыв_федераци_систер`
-2. `_шыв_65_çын_—_--_гр`
-3. `_юханшыв_шыв_-_ката`
+1. `шыв_шыв_—_венгрла._`
+2. `_шыв_федераци_агент`
+3. `_юханшыв_шыв_геоинф`
 
 
 ### Key Findings
 
 - **Best Predictability:** Context-4 (word) with 97.8% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (637,656 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (636,890 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,64 +346,64 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 148,629 |
-| Total Tokens | 3,880,417 |
-| Mean Frequency | 26.11 |
+| Vocabulary Size | 149,054 |
+| Total Tokens | 3,895,916 |
+| Mean Frequency | 26.14 |
 | Median Frequency | 4 |
-| Frequency Std Dev | 438.75 |
+| Frequency Std Dev | 439.39 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | шыв | 84,182 |
-| 2 | юханшыв | 53,747 |
-| 3 | в | 44,759 |
-| 4 | и | 40,900 |
-| 5 | с | 37,083 |
-| 6 | тата | 34,644 |
-| 7 | бассейн | 28,458 |
+| 1 | шыв | 84,160 |
+| 2 | юханшыв | 53,731 |
+| 3 | в | 45,242 |
+| 4 | и | 41,204 |
+| 5 | с | 37,543 |
+| 6 | тата | 34,625 |
+| 7 | бассейн | 28,455 |
 | 8 | км | 25,026 |
-| 9 | м | 24,683 |
-| 10 | рф | 24,443 |
+| 9 | м | 24,932 |
+| 10 | рф | 24,450 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | тӑтӑм | 2 |
-| 2 | мероприятире | 2 |
-| 3 | амфистрофпа | 2 |
-| 4 | ыйтрӗҫ | 2 |
-| 5 | поэмӑна | 2 |
-| 6 | хуламӑрта | 2 |
-| 7 | маршаленко | 2 |
-| 8 | шахмаметьев | 2 |
-| 9 | топилкин | 2 |
-| 10 | lupulella | 2 |
+| 1 | дустлик | 2 |
+| 2 | галляарал | 2 |
+| 3 | зарбдар | 2 |
+| 4 | джизакской | 2 |
+| 5 | сардоба | 2 |
+| 6 | баяут | 2 |
+| 7 | хаваст | 2 |
+| 8 | сырдарьинской | 2 |
+| 9 | пайт | 2 |
+| 10 | клинов | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 1.0392 |
-| R² (Goodness of Fit) | 0.997768 |
+| Zipf Coefficient | 1.0393 |
+| R² (Goodness of Fit) | 0.997747 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 30.1% |
+| Top 100 | 30.0% |
 | Top 1,000 | 56.1% |
 | Top 5,000 | 72.5% |
-| Top 10,000 | 79.1% |
+| Top 10,000 | 79.0% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9978 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 30.1% of corpus
-- **Long Tail:** 138,629 words needed for remaining 20.9% coverage
+- **Zipf Compliance:** R²=0.9977 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 30.0% of corpus
+- **Long Tail:** 139,054 words needed for remaining 21.0% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.8332 🏆 | 0.3551 | N/A | N/A |
-| **mono_64d** | 64 | 0.8307 | 0.2707 | N/A | N/A |
-| **mono_128d** | 128 | 0.8029 | 0.2175 | N/A | N/A |
+| **mono_32d** | 32 | 0.8326 🏆 | 0.3463 | N/A | N/A |
+| **mono_64d** | 64 | 0.8301 | 0.2835 | N/A | N/A |
+| **mono_128d** | 128 | 0.7992 | 0.2278 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8326 | 0.3575 | 0.0120 | 0.1340 |
+| **aligned_64d** | 64 | 0.8301 | 0.2722 | 0.0400 | 0.2360 |
+| **aligned_128d** | 128 | 0.7992 | 0.2219 | 0.0680 | 0.3000 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.8332 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.2811. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** mono_32d with 0.8326 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.2849. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 6.8% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **1.001** | High formulaic/idiomatic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -430,12 +465,12 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-а` | ыйтса, родства, мещерякова |
-| `-ен` | килнисен, центрӗнчен, теппермен |
-| `-ов` | камышлов, ярдов, горбов |
-| `-не` | мартинсоне, кёльне, характеристикине |
-| `-ем` | автомагистральсем, сооруженисем, алкалоидсем |
-| `-сем` | автомагистральсем, сооруженисем, алкалоидсем |
+| `-а` | курска, никсона, подвига |
+| `-ен` | америкасен, слышен, судьясен |
+| `-не` | взводне, очерксене, болгарине |
+| `-ов` | резюков, коршунов, щенков |
+| `-ем` | сикекенсем, символсем, перуанецсем |
+| `-ий` | выступлений, парфентий, праславянский |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -443,18 +478,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `олог` | 1.78x | 172 contexts | зоолог, эколог, геолог |
-| `сейн` | 2.68x | 24 contexts | хасейн, хусейн, басейн |
-| `рито` | 2.40x | 26 contexts | крито, ритон, барито |
-| `огра` | 1.72x | 94 contexts | богра, ноград, ограды |
-| `ссей` | 2.68x | 17 contexts | эссей, ессей, рассей |
-| `ншыв` | 2.63x | 17 contexts | юханшыв, юшаншыв, юханшыве |
-| `ерри` | 2.35x | 22 contexts | черри, дерри, шерри |
-| `исте` | 1.74x | 58 contexts | листе, истеми, листер |
-| `орин` | 1.61x | 74 contexts | шорин, горин, борин |
-| `аншы` | 2.63x | 13 contexts | юханшыв, юшаншыв, юханшыве |
-| `блик` | 2.15x | 17 contexts | облик, облике, облика |
-| `нист` | 1.75x | 30 contexts | финист, горнист, хронист |
+| `олог` | 2.08x | 173 contexts | геолог, пологи, эколог |
+| `сейн` | 2.92x | 24 contexts | сейнер, хусейн, хасейн |
+| `ссей` | 2.92x | 17 contexts | ессей, эссей, рассей |
+| `огра` | 1.78x | 95 contexts | богра, ограды, ограда |
+| `рито` | 2.46x | 26 contexts | ритон, крито, приток |
+| `ншыв` | 2.79x | 17 contexts | юшаншыв, юханшыв, юханшыве |
+| `ерри` | 2.45x | 22 contexts | черри, ферри, дерри |
+| `орин` | 1.72x | 74 contexts | дорин, шорин, борин |
+| `аншы` | 2.79x | 13 contexts | юшаншыв, юханшыв, юханшыве |
+| `исте` | 1.81x | 57 contexts | листе, хистет, истерн |
+| `блик` | 2.25x | 17 contexts | облик, облика, коблик |
+| `нист` | 1.86x | 30 contexts | финист, пианист, капнист |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -469,26 +504,28 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
-| арӑслансем | **`арӑслан-сем`** | 4.5 | `арӑслан` |
-| минералов | **`минерал-ов`** | 4.5 | `минерал` |
-| эскимосов | **`эскимос-ов`** | 4.5 | `эскимос` |
-| динамикине | **`динамики-не`** | 4.5 | `динамики` |
-| учрежденийӗсем | **`учрежденийӗ-сем`** | 4.5 | `учрежденийӗ` |
-| председательне | **`председатель-не`** | 4.5 | `председатель` |
-| флоренцине | **`флоренци-не`** | 4.5 | `флоренци` |
-| вестготсем | **`вестгот-сем`** | 4.5 | `вестгот` |
-| королевине | **`королеви-не`** | 4.5 | `королеви` |
-| паракансем | **`паракан-сем`** | 4.5 | `паракан` |
-| приоритетне | **`приоритет-не`** | 4.5 | `приоритет` |
-| юханшывен | **`юханшыв-ен`** | 4.5 | `юханшыв` |
-| йӑмпӑлчӑксем | **`йӑмпӑлчӑк-сем`** | 4.5 | `йӑмпӑлчӑк` |
-| пайланусем | **`пайлану-сем`** | 4.5 | `пайлану` |
-| асапланнине | **`асапланни-не`** | 4.5 | `асапланни` |
+| айсбергов | **`айсберг-ов`** | 4.5 | `айсберг` |
+| фахрутдинов | **`фахрутдин-ов`** | 4.5 | `фахрутдин` |
+| экономикине | **`экономики-не`** | 4.5 | `экономики` |
+| пурнӑҫланине | **`пурнӑҫлани-не`** | 4.5 | `пурнӑҫлани` |
+| ансамбльне | **`ансамбль-не`** | 4.5 | `ансамбль` |
+| хрустальне | **`хрусталь-не`** | 4.5 | `хрусталь` |
+| анатомине | **`анатоми-не`** | 4.5 | `анатоми` |
+| инженеров | **`инженер-ов`** | 4.5 | `инженер` |
+| багдасаров | **`багдасар-ов`** | 4.5 | `багдасар` |
+| фотографий | **`фотограф-ий`** | 4.5 | `фотограф` |
+| ассамблейине | **`ассамблейи-не`** | 4.5 | `ассамблейи` |
+| символикине | **`символики-не`** | 4.5 | `символики` |
+| бриллиантов | **`бриллиант-ов`** | 4.5 | `бриллиант` |
+| кинокритиков | **`кинокритик-ов`** | 4.5 | `кинокритик` |
+| наводнений | **`наводн-ен-ий`** | 3.0 | `наводн` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language CV appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Chuvash shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
+
+> **Note on Idiomaticity:** The high Idiomaticity Gap suggests a large number of frequent multi-word expressions or formulaic sequences that are statistically distinct from their component parts.
 
 ---
 ## 7. Summary & Recommendations
@@ -499,8 +536,8 @@ The language CV appears to be more isolating or has a highly fixed vocabulary. W
 
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
-| Tokenizer | **64k BPE** | Best compression (3.79x) |
-| N-gram | **2-gram** | Lowest perplexity (533) |
+| Tokenizer | **64k BPE** | Best compression (3.78x) |
+| N-gram | **2-gram** | Lowest perplexity (532) |
 | Markov | **Context-4** | Highest predictability (97.8%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
@@ -715,4 +752,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 11:00:29*
+*Report Date: 2026-01-03 23:50:11*

models/embeddings/aligned/cv_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:130680d2248f57ae1c36fc9ab1e6e0eda0a7d2308787a32f6d268a9423f78b02
+size 1096942926

models/embeddings/aligned/cv_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "cv", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/cv_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:695485e87ea1ae53971a035832321d9c5606a060d44573475d57fc389a4c22ce
+size 65664

models/embeddings/aligned/cv_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "cv",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 9371,
+  "vocab_size": 69656
+}

models/embeddings/aligned/cv_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cdc753305931cf408774c9aa08526d34b9612f3481c0d05a6e024ad61935fba
+size 275447118

models/embeddings/aligned/cv_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "cv", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/cv_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4367b3967495e33f979b6c501a98e9c8f8446e8335038b833c232be2fb9adfa8
+size 4224

models/embeddings/aligned/cv_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "cv",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 9371,
+  "vocab_size": 69656
+}

models/embeddings/aligned/cv_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97e63f8b115a259d1af448ee3adf665e39112df91110cabb11dc33e5522f85a6
+size 549279054

models/embeddings/aligned/cv_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "cv", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/cv_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d36328a9a375aba46f68707b77df6819a099e24dd96a52370a42a484641469cd
+size 16512

models/embeddings/aligned/cv_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "cv",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 9371,
+  "vocab_size": 69656
+}

models/embeddings/monolingual/cv_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6cb5a0b37d9ab0f0662eb5bebfad6713c0f0648b4a42c5b0743b36462becfd73
-size 1096617351
+oid sha256:130680d2248f57ae1c36fc9ab1e6e0eda0a7d2308787a32f6d268a9423f78b02
+size 1096942926

models/embeddings/monolingual/cv_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 69346
+  "vocab_size": 69656
 }

models/embeddings/monolingual/cv_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70007f295deadd64b95415e5970e57d83f6d92216140317dc2902789bf6da1de
-size 275359623
+oid sha256:8cdc753305931cf408774c9aa08526d34b9612f3481c0d05a6e024ad61935fba
+size 275447118

models/embeddings/monolingual/cv_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 69346
+  "vocab_size": 69656
 }

models/embeddings/monolingual/cv_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a5e77b5b9974a5a7347da624921ef4bd323fb01d62ffb62e50b9d2ca69a948e4
-size 549112199
+oid sha256:97e63f8b115a259d1af448ee3adf665e39112df91110cabb11dc33e5522f85a6
+size 549279054

models/embeddings/monolingual/cv_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 69346
+  "vocab_size": 69656
 }

models/subword_markov/cv_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e40043954b83a926cb69dce26f1f67caf4075d1babdc98a7a3543ca0afb6275b
-size 174799
+oid sha256:c3346d398d4005b40751df8506c1395d6f041d0b99c81d1c518e557a581ce287
+size 174960

models/subword_markov/cv_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "cv",
-  "unique_contexts": 3627,
-  "total_transitions": 29032785
+  "unique_contexts": 3635,
+  "total_transitions": 29154520
 }

models/subword_markov/cv_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da927115de7638fb21f20a2ffe9e1afb50762a8897417d080a3fbfbb98fede29
-size 1108384
+oid sha256:96e19271842803fd5df5bc0ebaa3af87e64fd6cbe51dbb84fbf1d48de884da92
+size 1112248

models/subword_markov/cv_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "cv",
-  "unique_contexts": 21896,
-  "total_transitions": 28978297
+  "unique_contexts": 21903,
+  "total_transitions": 29099815
 }

models/subword_markov/cv_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8cc4344364830518084929051b4ae7fb28c1adcdaa163605965b74168fdbce0e
-size 4746572
+oid sha256:51d6967be1a56e0bbb759070107c2ad61a4b36bb6f31f02a1953134b7eaf92e4
+size 4743832

models/subword_markov/cv_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "cv",
-  "unique_contexts": 135774,
-  "total_transitions": 28923809
+  "unique_contexts": 135543,
+  "total_transitions": 29045110
 }

models/subword_markov/cv_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bbad63be2ef01e2c8ae1ec9ea81d1bd588420f175327541dc01a0c9c80d33097
-size 16746581
+oid sha256:0836a4e42204ea786562096268d49a9fad133bda16cdbbad35bc4aa850e6ba44
+size 16755694

models/subword_markov/cv_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "cv",
-  "unique_contexts": 637656,
-  "total_transitions": 28869321
+  "unique_contexts": 636890,
+  "total_transitions": 28990405
 }

models/subword_ngram/cv_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a6dc007ee19af6fa94d26edbb4ded18e4913986e603f56c0eb0a613928d49afa
-size 111624
+oid sha256:cf921911e3ac02a40136c41c3cc4c4180d17b05f1c96145dc12d2f915ff427ad
+size 111747

models/subword_ngram/cv_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "cv",
-  "unique_ngrams": 7915,
-  "total_ngrams": 29032785
+  "unique_ngrams": 7908,
+  "total_ngrams": 29154520
 }

models/subword_ngram/cv_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a53062fcd868eabb3bedaef471787c15f34a1e4d7bd64541430cfad78fca8d3
-size 879358
+oid sha256:432c05f064a7e8b8652618f319730cbb81e70c2dfb55083be821ec9115b6dbe7
+size 878040

models/subword_ngram/cv_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "cv",
-  "unique_ngrams": 69443,
-  "total_ngrams": 28978297
+  "unique_ngrams": 69351,
+  "total_ngrams": 29099815
 }

models/subword_ngram/cv_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8ec5a505fdfed8bff6da064dc01c77eb56f326b342a958243b49281b2add80c1
-size 4733643
+oid sha256:3ab8068997235bcf52d4d43c2a5acd248a6d44be2e34b297dee57ac2a61e2939
+size 4727224

models/subword_ngram/cv_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "cv",
-  "unique_ngrams": 379120,
-  "total_ngrams": 28923809
+  "unique_ngrams": 378926,
+  "total_ngrams": 29045110
 }

models/subword_ngram/cv_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0536222fe93e097cd31d45bcd1240bb299e6063dc26a96c86a46b7ef8e3f541
+size 13563004

models/subword_ngram/cv_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "cv",
+  "unique_ngrams": 1007721,
+  "total_ngrams": 28990405
+}

models/tokenizer/cv_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47ce826b88b912a0ee22e1317e6f0258e7665b2a8c708987a2abda3b35abceff
-size 564983
+oid sha256:640e17b3c3cd055630d02fcfc5512991276e9037fb2a3e5d9930106f36d3504d
+size 564943

models/tokenizer/cv_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4f2de0762f3ba8f96fbc31f699f83e9b5dc454f42975ac96462ea30d55c9456f
-size 912506
+oid sha256:16b0ac2359544b071959cae6ac454d8543c07b671d234d4901fcaead35fd0b0d
+size 912892

models/tokenizer/cv_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5877d8c09381815bf1acb251e4535e07c5b0157e1b804e3dd591e8925c8268d9
-size 1629877
+oid sha256:54d00d4fc5c53f631ee8370abbab67b321b928713b3479bb02d4d6031446f747
+size 1631182

models/tokenizer/cv_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:64845d052561c0fd324adfbc8fd0d900a63c92141c555143df24fa74a9d7b46e
-size 398474
+oid sha256:97a9c78ba41c4fc4820e32fef12108c285f66f628f6c63d23cd3de9331e9428e
+size 398499

models/vocabulary/cv_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc7aca4a1d1d56f053bd0d6a123e4ef044a92c56295fcfe1be7fd13c9ca9e750
-size 2661487
+oid sha256:e837c9c3778915d77ff47bb28bd5e206258e5ffd40f04519456428cbd5bc0005
+size 2667305

models/vocabulary/cv_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "cv",
-  "vocabulary_size": 148629,
+  "vocabulary_size": 149054,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.08645308234335648,
+    "type_token_ratio": 0.08632064084275211,
     "coverage": {
-      "top_100": 0.28553243324755495,
-      "top_1000": 0.5333145325860036,
-      "top_5000": 0.6888109114837315,
-      "top_10000": 0.7510137225871395
+      "top_100": 0.2853232866356975,
+      "top_1000": 0.5330236414401892,
+      "top_5000": 0.6887052684199617,
+      "top_10000": 0.7508867673774949
     },
-    "hapax_count": 204527,
-    "hapax_ratio": 0.5791406630497571,
-    "total_documents": 54488
+    "hapax_count": 204934,
+    "hapax_ratio": 0.5789292292394093,
+    "total_documents": 54705
   }
 }

models/word_markov/cv_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c12896d7bb4321044e1d15a1f2c739138431e35a2c716c4bae85b032486c8b29
-size 23653393
+oid sha256:7c9a42680ec25e46b6e314a284536406b993337cb8b1c4daca1f33227ddec8e6
+size 23816242

models/word_markov/cv_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "cv",
-  "unique_contexts": 352008,
-  "total_transitions": 4030456
+  "unique_contexts": 352836,
+  "total_transitions": 4046145
 }

models/word_markov/cv_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04538a1a074396c3e2dba751a45663e0683f8dd3242406fc2c55ff9ab7c488ea
-size 58306430
+oid sha256:81e59ba49e08be0159d5763ef1440f0c2fbd0076b4cefac0d569414c9327edc5
+size 58581375

models/word_markov/cv_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "cv",
-  "unique_contexts": 1864001,
-  "total_transitions": 3975968
+  "unique_contexts": 1869675,
+  "total_transitions": 3991440
 }