Upload all models and assets for crh (latest)

.gitattributes

@@ -39,3 +39,5 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text
+visualizations/ngram_coverage.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: crh
-language_name: CRH
+language_name: Crimean Tatar
 language_family: turkic_kipchak
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-turkic_kipchak
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.773
+    value: 4.779
   - name: best_isotropy
     type: isotropy
-    value: 0.6920
+    value: 0.7031
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# CRH - Wikilangs Models
+# Crimean Tatar - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **CRH** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Crimean Tatar** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.643x | 3.65 | 0.2020% | 214,351 |
-| **16k** | 4.073x | 4.08 | 0.2258% | 191,723 |
-| **32k** | 4.453x | 4.46 | 0.2469% | 175,361 |
-| **64k** | 4.773x 🏆 | 4.78 | 0.2647% | 163,591 |
+| **8k** | 3.646x | 3.65 | 0.2038% | 212,471 |
+| **16k** | 4.078x | 4.08 | 0.2279% | 189,960 |
+| **32k** | 4.457x | 4.46 | 0.2492% | 173,772 |
+| **64k** | 4.779x 🏆 | 4.79 | 0.2672% | 162,079 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Şaburovo () - Rusiyeniñ Altay ülkesinde Solton rayonında bir qasaba. Ealisiniñ s...`
+**Sample 1:** `İslanovo () - Rusiyede, Başqırtistan Cumhuriyetiniñ Kuşnarenko rayonında bir köy...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁şab urovo ▁() ▁- ▁rusiyeniñ ▁altay ▁ülkesinde ▁solton ▁rayonında ▁bir ... (+12 more)` | 22 |
-| 16k | `▁şab urovo ▁() ▁- ▁rusiyeniñ ▁altay ▁ülkesinde ▁solton ▁rayonında ▁bir ... (+12 more)` | 22 |
-| 32k | `▁şab urovo ▁() ▁- ▁rusiyeniñ ▁altay ▁ülkesinde ▁solton ▁rayonında ▁bir ... (+12 more)` | 22 |
-| 64k | `▁şaburovo ▁() ▁- ▁rusiyeniñ ▁altay ▁ülkesinde ▁solton ▁rayonında ▁bir ▁qasaba ... (+11 more)` | 21 |
+| 8k | `▁İs lan ovo ▁() ▁- ▁rusiyede , ▁başqırtistan ▁cumhuriyetiniñ ▁kuşnarenko ... (+13 more)` | 23 |
+| 16k | `▁İs lanovo ▁() ▁- ▁rusiyede , ▁başqırtistan ▁cumhuriyetiniñ ▁kuşnarenko ▁rayonında ... (+12 more)` | 22 |
+| 32k | `▁İs lanovo ▁() ▁- ▁rusiyede , ▁başqırtistan ▁cumhuriyetiniñ ▁kuşnarenko ▁rayonında ... (+12 more)` | 22 |
+| 64k | `▁İslanovo ▁() ▁- ▁rusiyede , ▁başqırtistan ▁cumhuriyetiniñ ▁kuşnarenko ▁rayonında ▁bir ... (+11 more)` | 21 |
 
-**Sample 2:** `Polzunovo () - Rusiyeniñ Altay ülkesinde Barnaul şeer bölgesinda bir stantsiya. ...`
+**Sample 2:** `Drujbivka () - Ukrainanıñ Jıtomır vilâyetinde Korosten rayonında bir köy. Ealisi...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁pol z unovo ▁() ▁- ▁rusiyeniñ ▁altay ▁ülkesinde ▁barnaul ▁şeer ... (+17 more)` | 27 |
-| 16k | `▁pol z unovo ▁() ▁- ▁rusiyeniñ ▁altay ▁ülkesinde ▁barnaul ▁şeer ... (+17 more)` | 27 |
-| 32k | `▁pol z unovo ▁() ▁- ▁rusiyeniñ ▁altay ▁ülkesinde ▁barnaul ▁şeer ... (+17 more)` | 27 |
-| 64k | `▁polzunovo ▁() ▁- ▁rusiyeniñ ▁altay ▁ülkesinde ▁barnaul ▁şeer ▁bölgesinda ▁bir ... (+15 more)` | 25 |
+| 8k | `▁druj bivka ▁() ▁- ▁ukrainanıñ ▁jıtomır ▁vilâyetinde ▁korosten ▁rayonında ▁bir ... (+12 more)` | 22 |
+| 16k | `▁druj bivka ▁() ▁- ▁ukrainanıñ ▁jıtomır ▁vilâyetinde ▁korosten ▁rayonında ▁bir ... (+12 more)` | 22 |
+| 32k | `▁druj bivka ▁() ▁- ▁ukrainanıñ ▁jıtomır ▁vilâyetinde ▁korosten ▁rayonında ▁bir ... (+12 more)` | 22 |
+| 64k | `▁drujbivka ▁() ▁- ▁ukrainanıñ ▁jıtomır ▁vilâyetinde ▁korosten ▁rayonında ▁bir ▁köy ... (+11 more)` | 21 |
 
-**Sample 3:** `Bobliv () - Ukrainanıñ Vinnıtsâ vilâyetinde Vinnıtsâ rayonında bir köy. Ealisini...`
+**Sample 3:** `Koltunovka () - Rusiyeniñ Belgorod vilâyetinde, Alekseyevka rayonında bir köy. E...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁bob liv ▁() ▁- ▁ukrainanıñ ▁vinnıtsâ ▁vilâyetinde ▁vinnıtsâ ▁rayonında ▁bir ... (+12 more)` | 22 |
-| 16k | `▁bob liv ▁() ▁- ▁ukrainanıñ ▁vinnıtsâ ▁vilâyetinde ▁vinnıtsâ ▁rayonında ▁bir ... (+12 more)` | 22 |
-| 32k | `▁bob liv ▁() ▁- ▁ukrainanıñ ▁vinnıtsâ ▁vilâyetinde ▁vinnıtsâ ▁rayonında ▁bir ... (+12 more)` | 22 |
-| 64k | `▁bobliv ▁() ▁- ▁ukrainanıñ ▁vinnıtsâ ▁vilâyetinde ▁vinnıtsâ ▁rayonında ▁bir ▁köy ... (+11 more)` | 21 |
+| 8k | `▁kol tun ovka ▁() ▁- ▁rusiyeniñ ▁belgorod ▁vilâyetinde , ▁alekseyevka ... (+15 more)` | 25 |
+| 16k | `▁kol tun ovka ▁() ▁- ▁rusiyeniñ ▁belgorod ▁vilâyetinde , ▁alekseyevka ... (+15 more)` | 25 |
+| 32k | `▁kol tun ovka ▁() ▁- ▁rusiyeniñ ▁belgorod ▁vilâyetinde , ▁alekseyevka ... (+15 more)` | 25 |
+| 64k | `▁koltunovka ▁() ▁- ▁rusiyeniñ ▁belgorod ▁vilâyetinde , ▁alekseyevka ▁rayonında ▁bir ... (+13 more)` | 23 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.773x compression
-- **Lowest UNK Rate:** 8k with 0.2020% unknown tokens
+- **Best Compression:** 64k achieves 4.779x compression
+- **Lowest UNK Rate:** 8k with 0.2038% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 851 | 9.73 | 10,225 | 56.0% | 74.3% |
-| **2-gram** | Subword | 349 🏆 | 8.45 | 3,905 | 63.4% | 98.0% |
-| **3-gram** | Word | 1,276 | 10.32 | 13,312 | 49.1% | 71.8% |
-| **3-gram** | Subword | 2,227 | 11.12 | 29,302 | 33.0% | 71.7% |
-| **4-gram** | Word | 4,192 | 12.03 | 31,529 | 31.9% | 54.7% |
-| **4-gram** | Subword | 7,868 | 12.94 | 131,434 | 26.0% | 52.2% |
+| **2-gram** | Word | 849 | 9.73 | 10,213 | 56.1% | 74.4% |
+| **2-gram** | Subword | 348 🏆 | 8.44 | 3,878 | 63.4% | 98.0% |
+| **3-gram** | Word | 1,276 | 10.32 | 13,301 | 49.1% | 71.8% |
+| **3-gram** | Subword | 2,220 | 11.12 | 29,221 | 33.1% | 71.8% |
+| **4-gram** | Word | 4,190 | 12.03 | 31,513 | 31.9% | 54.7% |
+| **4-gram** | Subword | 7,833 | 12.94 | 131,199 | 26.0% | 52.3% |
+| **5-gram** | Word | 6,061 | 12.57 | 29,487 | 24.1% | 48.5% |
+| **5-gram** | Subword | 16,690 | 14.03 | 285,107 | 23.4% | 46.1% |
 
 ### Top 5 N-grams by Size
 
@@ -150,19 +162,19 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `ealisiniñ sayısı` | 20,731 |
-| 2 | `rayonında bir` | 17,343 |
+| 1 | `ealisiniñ sayısı` | 20,740 |
+| 2 | `rayonında bir` | 17,352 |
 | 3 | `meskün yerler` | 12,883 |
-| 4 | `bir köy` | 10,053 |
-| 5 | `köy ealisiniñ` | 9,130 |
+| 4 | `bir köy` | 10,061 |
+| 5 | `köy ealisiniñ` | 9,139 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `rayonında bir köy` | 9,305 |
-| 2 | `köy ealisiniñ sayısı` | 9,130 |
-| 3 | `bir köy ealisiniñ` | 9,130 |
+| 1 | `rayonında bir köy` | 9,314 |
+| 2 | `bir köy ealisiniñ` | 9,139 |
+| 3 | `köy ealisiniñ sayısı` | 9,139 |
 | 4 | `rayonındaki meskün yerler` | 5,591 |
 | 5 | `kişi meskün yerler` | 4,604 |
 
@@ -170,48 +182,68 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `bir köy ealisiniñ sayısı` | 9,130 |
-| 2 | `rayonında bir köy ealisiniñ` | 8,976 |
+| 1 | `bir köy ealisiniñ sayısı` | 9,139 |
+| 2 | `rayonında bir köy ealisiniñ` | 8,985 |
 | 3 | `bir köydir ealisiniñ sayısı` | 4,601 |
 | 4 | `rayonında bir köydir ealisiniñ` | 4,565 |
 | 5 | `i̇htar rayonındaki meskün yerler` | 3,615 |
 
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `rayonında bir köy ealisiniñ sayısı` | 8,985 |
+| 2 | `rayonında bir köydir ealisiniñ sayısı` | 4,565 |
+| 3 | `kişi i̇htar rayonındaki meskün yerler` | 2,558 |
+| 4 | `asırnıñ bir senesi vaqialar doğumlar` | 1,996 |
+| 5 | `bir senesi vaqialar doğumlar ölümler` | 1,917 |
+
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `i n` | 101,180 |
-| 2 | `e r` | 95,484 |
-| 3 | `a _` | 88,670 |
-| 4 | `r _` | 84,656 |
-| 5 | `. _` | 80,946 |
+| 1 | `i n` | 101,089 |
+| 2 | `e r` | 95,398 |
+| 3 | `a _` | 88,613 |
+| 4 | `r _` | 84,598 |
+| 5 | `. _` | 80,856 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `i ñ _` | 43,476 |
-| 2 | `n i ñ` | 42,980 |
-| 3 | `l e r` | 42,878 |
-| 4 | `n d e` | 35,841 |
-| 5 | `e t i` | 35,623 |
+| 1 | `i ñ _` | 43,406 |
+| 2 | `n i ñ` | 42,914 |
+| 3 | `l e r` | 42,891 |
+| 4 | `n d e` | 35,848 |
+| 5 | `e t i` | 35,643 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `n i ñ _` | 42,723 |
-| 2 | `i n d e` | 34,201 |
-| 3 | `y e t i` | 30,809 |
-| 4 | `ı n d a` | 30,136 |
-| 5 | `_ b i r` | 29,657 |
+| 1 | `n i ñ _` | 42,657 |
+| 2 | `i n d e` | 34,217 |
+| 3 | `y e t i` | 30,830 |
+| 4 | `ı n d a` | 30,087 |
+| 5 | `_ b i r` | 29,643 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `i n i ñ _` | 28,194 |
+| 2 | `y e t i n` | 28,057 |
+| 3 | `_ b i r _` | 27,628 |
+| 4 | `r a y o n` | 26,921 |
+| 5 | `_ r a y o` | 26,900 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram (subword) with 349
+- **Best Perplexity:** 2-gram (subword) with 348
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~52% of corpus
+- **Coverage:** Top-1000 patterns cover ~46% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.6260 | 1.543 | 3.00 | 128,781 | 37.4% |
-| **1** | Subword | 0.8874 | 1.850 | 6.86 | 1,506 | 11.3% |
-| **2** | Word | 0.1303 | 1.094 | 1.24 | 384,958 | 87.0% |
-| **2** | Subword | 0.9042 | 1.872 | 5.57 | 10,319 | 9.6% |
-| **3** | Word | 0.0387 | 1.027 | 1.07 | 475,888 | 96.1% |
-| **3** | Subword | 0.8152 | 1.760 | 3.87 | 57,491 | 18.5% |
-| **4** | Word | 0.0241 🏆 | 1.017 | 1.05 | 504,736 | 97.6% |
-| **4** | Subword | 0.6067 | 1.523 | 2.54 | 222,424 | 39.3% |
+| **1** | Word | 0.6244 | 1.542 | 2.99 | 128,666 | 37.6% |
+| **1** | Subword | 0.8852 | 1.847 | 6.85 | 1,505 | 11.5% |
+| **2** | Word | 0.1302 | 1.094 | 1.24 | 383,467 | 87.0% |
+| **2** | Subword | 0.9025 | 1.869 | 5.57 | 10,300 | 9.7% |
+| **3** | Word | 0.0387 | 1.027 | 1.07 | 474,016 | 96.1% |
+| **3** | Subword | 0.8153 | 1.760 | 3.87 | 57,358 | 18.5% |
+| **4** | Word | 0.0242 🏆 | 1.017 | 1.05 | 502,796 | 97.6% |
+| **4** | Subword | 0.6069 | 1.523 | 2.54 | 221,948 | 39.3% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `bir köy ealisiniñ sayısı 102 1 ya ayaqlı bir köydir ealisiniñ sayısı kişi vilâyetindeki qasabalar ma...`
-2. `kişi vilâyetindeki meskün yerleri şeer şeklinde qasabalar albörikent accı suv miktarınıñ az olğanına...`
-3. `sayısı 919 kişi vilâyetindeki şeerler boksitogorsk rayonında bir dürki türkiyede oturğan 2 567 kişi ...`
+1. `bir cemaatı ukrainanıñ jıtomır vilâyetinde olevsk rayonında bir şeer şeklinde qasabalar vahruşev nog...`
+2. `kişi rayonındaki meskün yerler köyler abatskoye rusiyeniñ hantı mansi muhtar cumhuriyetinıñ devlet g...`
+3. `sayısı 0 kişi meskün yerler veloturizm iklim deñişmelerine çoq yüklü yükni yükniñ yüksek mölekulâr o...`
 
 **Context Size 2:**
 
-1. `ealisiniñ sayısı 3 900 kişi muhtar vilâyetindeki şeer şeklinde qasabalar bazarnıy karabulak rayonını...`
-2. `rayonında bir şeer gornozavodsk rayonınıñ merkezi ealisiniñ sayısı 562 kişi vilâyetindeki köyler ray...`
-3. `bir köy ealisiniñ sayısı 952 kişi vilâyetindeki meskün yerler köyler atıflar rayonındaki meskün yerl...`
+1. `ealisiniñ sayısı kişi senesi vilâyetindeki qasabalar`
+2. `rayonında bir aul adıge habl calancük kiçik i̇ncik kavkazskiy pregradna üçköken habez erkin şeer rus...`
+3. `bir köy oktâbr rayonınıñ merkezi ealisiniñ sayısı 202 939 kişi senesi atıflar rayonındaki meskün yer...`
 
 **Context Size 3:**
 
-1. `rayonında bir köy ealisiniñ sayısı kişi i̇htar rayonındaki meskün yerler köyler atıflar rayonındaki ...`
-2. `köy ealisiniñ sayısı 183 kişi i̇htar rayonındaki meskün yerler köyler atıflar rayonındaki meskün yer...`
-3. `bir köy ealisiniñ sayısı 7 kişi i̇htar rayonındaki meskün yerler şeer şeklinde qasabalar verhovye gl...`
+1. `rayonında bir köy ealisiniñ sayısı 394 kişi senesi atıflar rayonındaki meskün yerler köyler atıflar ...`
+2. `bir köy ealisiniñ sayısı 593 kişi i̇htar rayonındaki meskün yerler köyler atıflar rayonındaki meskün...`
+3. `köy ealisiniñ sayısı 828 kişi vilâyetindeki meskün yerler`
 
 **Context Size 4:**
 
-1. `bir köy ealisiniñ sayısı kişi vilâyetindeki şeer şeklinde qasabalar şeer bölgesindeki meskün yerler ...`
-2. `rayonında bir köy ealisiniñ sayısı 572 kişi i̇htar rayonındaki meskün yerler vilâyetindeki şeer şekl...`
-3. `bir köydir ealisiniñ sayısı 306 kişi meskün yerler`
+1. `bir köy ealisiniñ sayısı kişi vilâyetindeki meskün yerler`
+2. `rayonında bir köy ealisiniñ sayısı 134 kişi vilâyetindeki meskün yerler`
+3. `bir köydir ealisiniñ sayısı 25 kişi i̇htar rayonındaki meskün yerler vilâyetindeki şeer şeklinde qas...`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `_revıñ_moviñmur_`
-2. `a_()._ay.._obeto`
-3. `i_робу_yıñ_()_fi`
+1. `_()_-_qmı_mi._be`
+2. `ariraye_altviyür`
+3. `i._bişekayay._()`
 
 **Context Size 2:**
 
-1. `inovaq_ümalmannıñ`
-2. `a_başqırınde_talo`
-3. `r_rus_graynay_the`
+1. `iniv-ufterlar,_ad`
+2. `a_balisiyentılari`
+3. `r_rusiyetingrayıs`
 
 **Context Size 3:**
 
-1. `iñ_brânskaya_yañız`
-2. `niñ_araman_sürtüyl`
-3. `nde_bek-tatar_vilâ`
+1. `iñ_sayısı_591_belg`
+2. `niñ_sayısı_kir._ea`
+3. `nde_dinde_ögrendi_`
 
 **Context Size 4:**
 
-1. `niñ_stepanovskiy_—_`
-2. `inde_ölümlerinen_so`
-3. `yetindeki_meskün_ye`
+1. `niñ_noviçi_bar._cev`
+2. `inde_kontsev_artemi`
+3. `yetinde_bir_qast_ma`
 
 
 ### Key Findings
 
 - **Best Predictability:** Context-4 (word) with 97.6% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (222,424 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (221,948 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,64 +346,64 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 51,581 |
-| Total Tokens | 778,307 |
+| Vocabulary Size | 51,458 |
+| Total Tokens | 776,471 |
 | Mean Frequency | 15.09 |
 | Median Frequency | 3 |
-| Frequency Std Dev | 271.68 |
+| Frequency Std Dev | 272.01 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | bir | 27,780 |
-| 2 | kişi | 20,845 |
-| 3 | sayısı | 20,811 |
-| 4 | ealisiniñ | 20,761 |
-| 5 | rayonında | 17,383 |
+| 1 | bir | 27,753 |
+| 2 | kişi | 20,857 |
+| 3 | sayısı | 20,821 |
+| 4 | ealisiniñ | 20,770 |
+| 5 | rayonında | 17,392 |
 | 6 | meskün | 13,506 |
-| 7 | yerler | 12,928 |
-| 8 | vilâyetinde | 12,431 |
-| 9 | köy | 10,895 |
+| 7 | yerler | 12,926 |
+| 8 | vilâyetinde | 12,440 |
+| 9 | köy | 10,901 |
 | 10 | rusiyeniñ | 9,597 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | ekranlar | 2 |
-| 2 | oem | 2 |
-| 3 | macbook | 2 |
-| 4 | mahsuldarlıq | 2 |
-| 5 | planşetler | 2 |
-| 6 | fatemeh | 2 |
-| 7 | movaghar | 2 |
-| 8 | پریسا | 2 |
-| 9 | موقر | 2 |
-| 10 | slammer | 2 |
+| 1 | зияде | 2 |
+| 2 | atalarnıñ | 2 |
+| 3 | kotsubınskıylar | 2 |
+| 4 | yüneskonıñ | 2 |
+| 5 | دیللر | 2 |
+| 6 | ازبری | 2 |
+| 7 | اولان | 2 |
+| 8 | سامانچی | 2 |
+| 9 | قیزی | 2 |
+| 10 | samançı | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 0.9849 |
-| R² (Goodness of Fit) | 0.998059 |
+| Zipf Coefficient | 0.9856 |
+| R² (Goodness of Fit) | 0.998043 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 45.4% |
-| Top 1,000 | 63.7% |
-| Top 5,000 | 78.1% |
-| Top 10,000 | 84.3% |
+| Top 100 | 45.6% |
+| Top 1,000 | 63.8% |
+| Top 5,000 | 78.2% |
+| Top 10,000 | 84.4% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9981 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 45.4% of corpus
-- **Long Tail:** 41,581 words needed for remaining 15.7% coverage
+- **Zipf Compliance:** R²=0.9980 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 45.6% of corpus
+- **Long Tail:** 41,458 words needed for remaining 15.6% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.6920 🏆 | 0.3816 | N/A | N/A |
-| **mono_64d** | 64 | 0.4424 | 0.3546 | N/A | N/A |
-| **mono_128d** | 128 | 0.1085 | 0.3496 | N/A | N/A |
+| **mono_32d** | 32 | 0.7031 🏆 | 0.3722 | N/A | N/A |
+| **mono_64d** | 64 | 0.4233 | 0.3424 | N/A | N/A |
+| **mono_128d** | 128 | 0.1068 | 0.3377 | N/A | N/A |
+| **aligned_32d** | 32 | 0.7031 | 0.3786 | 0.0140 | 0.1600 |
+| **aligned_64d** | 64 | 0.4233 | 0.3386 | 0.0380 | 0.2140 |
+| **aligned_128d** | 128 | 0.1068 | 0.3419 | 0.0560 | 0.2680 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.6920 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.3619. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** mono_32d with 0.7031 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.3519. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 5.6% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **0.052** | Low formulaic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -430,14 +465,14 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-a` | belâyeva, sira, boynuna |
-| `-ka` | tatyanovka, verigovka, mazanka |
-| `-vo` | çkalovo, beketovo, çufarovo |
-| `-vka` | tatyanovka, verigovka, karnauhovka |
-| `-an` | yasaqlağan, başqırtistan, i̇talyan |
-| `-ovo` | çkalovo, beketovo, çufarovo |
-| `-en` | yerlerinden, esitgen, yaratıcılığınen |
-| `-ya` | nesterovskaya, podsosennaya, borzovaya |
+| `-a` | terehova, biçura, observatoriya |
+| `-ka` | novosölka, alekseyevka, kapustânka |
+| `-vo` | korolövo, semenovo, hetovo |
+| `-vka` | alekseyevka, dolgalovka, svetlovka |
+| `-an` | turan, birobican, adlandırğan |
+| `-ovo` | semenovo, hetovo, panfilovo |
+| `-ye` | zapolnoye, smelıye, voznesenskoye |
+| `-en` | keçirmegen, nevbetten, neogen |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -445,18 +480,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `rler` | 1.64x | 57 contexts | erler, yerler, kirler |
-| `siye` | 2.07x | 21 contexts | asiye, rusiye, vasiyet |
-| `isin` | 1.60x | 34 contexts | lisin, episine, ekisini |
-| `iniñ` | 1.68x | 26 contexts | eliniñ, aliniñ, öziniñ |
-| `nesi` | 1.66x | 22 contexts | nesir, nesib, nesil |
-| `usiy` | 2.15x | 9 contexts | lusiya, rusiye, hususiy |
-| `eniñ` | 1.77x | 15 contexts | heniñ, seniñ, ekeniñ |
-| `lâye` | 1.89x | 11 contexts | gulâyev, belâyev, vilâyet |
-| `âyet` | 1.89x | 11 contexts | vilâyet, menâyet, şikâyet |
-| `yeti` | 1.62x | 17 contexts | yetim, yetip, yetişe |
-| `sini` | 1.62x | 15 contexts | siniy, sinip, sesini |
-| `tind` | 1.75x | 11 contexts | etinden, betinde, şetinde |
+| `leri` | 1.60x | 110 contexts | ileri, lerik, galeri |
+| `rler` | 1.60x | 57 contexts | erler, yerler, derler |
+| `siye` | 2.05x | 21 contexts | asiye, rusiye, tevsiye |
+| `isin` | 1.57x | 31 contexts | episine, kerisin, reisini |
+| `iniñ` | 1.64x | 26 contexts | eviniñ, iliniñ, eliniñ |
+| `nesi` | 1.64x | 22 contexts | nesib, nesil, nesir |
+| `eniñ` | 1.75x | 16 contexts | seniñ, heniñ, ekeniñ |
+| `usiy` | 2.11x | 9 contexts | lusiya, rusiye, hususiy |
+| `lâye` | 1.87x | 11 contexts | belâyev, gulâyev, vilâyet |
+| `âyet` | 1.87x | 11 contexts | menâyet, vilâyet, şikâyet |
+| `sini` | 1.70x | 14 contexts | siniy, sinip, aksini |
+| `yeti` | 1.59x | 17 contexts | yetip, yetim, yetişe |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -471,26 +506,26 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
-| çabayevka | **`çaba-ye-vka`** | 6.0 | `çaba` |
-| ruzayevka | **`ruza-ye-vka`** | 6.0 | `ruza` |
-| turmayevo | **`turma-ye-vo`** | 6.0 | `turma` |
-| natalivka | **`natali-vka`** | 4.5 | `natali` |
-| çingizovo | **`çingiz-ovo`** | 4.5 | `çingiz` |
-| krasnoyarovo | **`krasnoyar-ovo`** | 4.5 | `krasnoyar` |
-| kapustinka | **`kapustin-ka`** | 4.5 | `kapustin` |
-| malinovka | **`malino-vka`** | 4.5 | `malino` |
-| soldatovo | **`soldat-ovo`** | 4.5 | `soldat` |
-| kaltımanovo | **`kaltım-an-ovo`** | 3.0 | `kaltım` |
-| balabanovo | **`balab-an-ovo`** | 3.0 | `balab` |
-| olehovskaya | **`olehovs-ka-ya`** | 3.0 | `olehovs` |
-| olşanskaya | **`olşans-ka-ya`** | 3.0 | `olşans` |
-| kıtmanovo | **`kıtm-an-ovo`** | 3.0 | `kıtm` |
-| kropıvenka | **`kropıv-en-ka`** | 3.0 | `kropıv` |
+| gazetanen | **`gazet-an-en`** | 6.0 | `gazet` |
+| ananyevka | **`anan-ye-vka`** | 6.0 | `anan` |
+| petrusnıñ | **`petrus-nıñ`** | 4.5 | `petrus` |
+| vesiqalarınıñ | **`vesiqaları-nıñ`** | 4.5 | `vesiqaları` |
+| nikiforovo | **`nikifor-ovo`** | 4.5 | `nikifor` |
+| sistemasınıñ | **`sisteması-nıñ`** | 4.5 | `sisteması` |
+| qısımlarınıñ | **`qısımları-nıñ`** | 4.5 | `qısımları` |
+| borispolye | **`borispol-ye`** | 4.5 | `borispol` |
+| programmanıñ | **`programma-nıñ`** | 4.5 | `programma` |
+| gotlarnıñ | **`gotlar-nıñ`** | 4.5 | `gotlar` |
+| qadılıqnıñ | **`qadılıq-nıñ`** | 4.5 | `qadılıq` |
+| kopelânka | **`kopelân-ka`** | 4.5 | `kopelân` |
+| mahsulatlarnıñ | **`mahsulatlar-nıñ`** | 4.5 | `mahsulatlar` |
+| nigeriyanıñ | **`nigeriya-nıñ`** | 4.5 | `nigeriya` |
+| qasabanıñ | **`qasaba-nıñ`** | 4.5 | `qasaba` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language CRH appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Crimean Tatar shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
 
 ---
 ## 7. Summary & Recommendations
@@ -501,8 +536,8 @@ The language CRH appears to be more isolating or has a highly fixed vocabulary.
 
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
-| Tokenizer | **64k BPE** | Best compression (4.77x) |
-| N-gram | **2-gram** | Lowest perplexity (349) |
+| Tokenizer | **64k BPE** | Best compression (4.78x) |
+| N-gram | **2-gram** | Lowest perplexity (348) |
 | Markov | **Context-4** | Highest predictability (97.6%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
@@ -717,4 +752,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 10:33:02*
+*Report Date: 2026-01-03 20:48:59*

models/embeddings/aligned/crh_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b25c442c888bd49ba33657541b259bc8c7993a4a822d0b4f047f95e2414bb7ba
+size 1039762899

models/embeddings/aligned/crh_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "crh", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/crh_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0c9317a0ebc1d09f0f1eb19e67a7a3a45f82bead865a7780074382f6429021a
+size 65664

models/embeddings/aligned/crh_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "crh",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 3418,
+  "vocab_size": 15135
+}

models/embeddings/aligned/crh_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c71dd6902c7a34a2394c9007bfb8e46171666b35fc184eb1e5c00d5b0223a0e3
+size 260139219

models/embeddings/aligned/crh_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "crh", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/crh_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:667941779b81d220b32db7a7da75e1b2dd17098b578c084166c8a3b42bcacb01
+size 4224

models/embeddings/aligned/crh_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "crh",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 3418,
+  "vocab_size": 15135
+}

models/embeddings/aligned/crh_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6078827c3fe5f982b19139f13fe285e0abd69c77e62cd8f4ffa32fae7fc79c56
+size 520013779

models/embeddings/aligned/crh_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "crh", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/crh_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:613d15270964a5f77a2cf44a9f7b6195546589f19844110624f808558bcfbfc6
+size 16512

models/embeddings/aligned/crh_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "crh",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 3418,
+  "vocab_size": 15135
+}

models/embeddings/monolingual/crh_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2c03c90e9321eff6a486a7b259eb6d14069ad97d6e4851aaa1d32c01bb7310b
-size 1039804561
+oid sha256:b25c442c888bd49ba33657541b259bc8c7993a4a822d0b4f047f95e2414bb7ba
+size 1039762899

models/embeddings/monolingual/crh_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 15175
+  "vocab_size": 15135
 }

models/embeddings/monolingual/crh_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:08fc415d5a1b109301770bd6dbcbfbdde2008dd25a25d23f81ace78bc3971a3a
-size 260150161
+oid sha256:c71dd6902c7a34a2394c9007bfb8e46171666b35fc184eb1e5c00d5b0223a0e3
+size 260139219

models/embeddings/monolingual/crh_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 15175
+  "vocab_size": 15135
 }

models/embeddings/monolingual/crh_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e5d8136014d664519e3bafeb0cfc668b4993f78425d51bdcf0d9d06734bd0726
-size 520034961
+oid sha256:6078827c3fe5f982b19139f13fe285e0abd69c77e62cd8f4ffa32fae7fc79c56
+size 520013779

models/embeddings/monolingual/crh_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 15175
+  "vocab_size": 15135
 }

models/subword_markov/crh_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2ba0d156d7362837baf91177f2123bb0ccb074f4556c38c22f0d046b19f9805
-size 84372
+oid sha256:fd8b34f078483c036527908f67f3c5610d2c29f649b2720f530fa2dfa1b0e8db
+size 85168

models/subword_markov/crh_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "crh",
-  "unique_contexts": 1506,
-  "total_transitions": 6777473
+  "unique_contexts": 1505,
+  "total_transitions": 6765123
 }

models/subword_markov/crh_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a42c66f0a162aa7d06896db540b1636552e8a9ebc256f5e6cf8d91c7301d5444
-size 481861
+oid sha256:4761755ce745fbee84031c20a4990ec524aad21f4e11fd88cdf2c94a677ff81a
+size 483634

models/subword_markov/crh_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "crh",
-  "unique_contexts": 10319,
-  "total_transitions": 6748084
+  "unique_contexts": 10300,
+  "total_transitions": 6735724
 }

models/subword_markov/crh_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a42665c99cf0102347580ad03ad8662c7dcd958de7c0259e8380dcbc939a89f
-size 1701410
+oid sha256:0d7b982a5cdc24199071b45ac69453cfed21172bca4664510b3570bd6288d358
+size 1694801

models/subword_markov/crh_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "crh",
-  "unique_contexts": 57491,
-  "total_transitions": 6718695
+  "unique_contexts": 57358,
+  "total_transitions": 6706325
 }

models/subword_markov/crh_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:769a741cb6357905e7c73c65cde5f662a2cbc84d3c2b1870ec910617615ba5a1
-size 4686904
+oid sha256:903e403063a2e04972c716a4a97faa64eeb890febdf6661d0331d774f610cf77
+size 4695561

models/subword_markov/crh_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "crh",
-  "unique_contexts": 222424,
-  "total_transitions": 6689306
+  "unique_contexts": 221948,
+  "total_transitions": 6676926
 }

models/subword_ngram/crh_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d352e2e3b7dd102bd7055d607a37c3738aff67c0c66285e95804ad330500d280
-size 52920
+oid sha256:c38e7c0b1e3acc2977ee1a5653edebd3b316b0ecf31ef015fc3403af3c4d4525
+size 52765

models/subword_ngram/crh_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "crh",
-  "unique_ngrams": 3905,
-  "total_ngrams": 6777473
+  "unique_ngrams": 3878,
+  "total_ngrams": 6765123
 }

models/subword_ngram/crh_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a92a6f76ef54f7e43af0d7e9c762ace522be6627fa4f29ab3df5200c1f2a63d
-size 375416
+oid sha256:2a4981bac1848719016e59cfd08d405c69d980728a0f727aa6f9435123b26a87
+size 373904

models/subword_ngram/crh_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "crh",
-  "unique_ngrams": 29302,
-  "total_ngrams": 6748084
+  "unique_ngrams": 29221,
+  "total_ngrams": 6735724
 }

models/subword_ngram/crh_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ddc20df05c5929fbc17c37bba729a38c20ac69752548221a46c1e42b8e8359f
-size 1562486
+oid sha256:1e162daa1f3626c601e1d0d9e186afe361a7214b21cdb31e6e38b1c0ff426e87
+size 1557256

models/subword_ngram/crh_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "crh",
-  "unique_ngrams": 131434,
-  "total_ngrams": 6718695
+  "unique_ngrams": 131199,
+  "total_ngrams": 6706325
 }

models/subword_ngram/crh_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd6cb712bbfb90b8c57051a1a6ab29d11883ab8c6101589eac0e5c771904d749
+size 3420383

models/subword_ngram/crh_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "crh",
+  "unique_ngrams": 285107,
+  "total_ngrams": 6676926
+}

models/tokenizer/crh_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7252484520841503b3c10a110b3ea4ba3a62f7073a69a342466526a0b31ef51c
-size 513301
+oid sha256:2b38599300ad9ca2774c4e30571d5b396d17f74e01d4c4ea9e8802702762bfca
+size 513237

models/tokenizer/crh_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:60b38160d9a927f6ea17a75ef13bbe991a80a50121d1c845806c967c815488f3
-size 797422
+oid sha256:89ef3e7c2f709f2492356b7e01b86aedb9ee84f18164cfb4e53e4be44c2c6b60
+size 797628

models/tokenizer/crh_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b45034637ab90c0741ce2ac1009165fc7e87f960f1ff87e489cabdedb42a8c8
-size 1425086
+oid sha256:ec779410f4c4357761af49a2ef1889928e20b37023389f9e39431309315493ba
+size 1425429

models/tokenizer/crh_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:414811c3993e9f6351075aaf47ac4ebe0185a4a1e2948c911bbbe25a0c48d8d1
-size 373959
+oid sha256:988e38ab80cfb7b6ebaf34b1f797a0205dafd67a8df63b461a940f9874d2ac2c
+size 374054

models/vocabulary/crh_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e154d72a53b3ecac61e12ba483392cfb82a4cc6a0a1b854df1b5a9a58503b2e9
-size 892791
+oid sha256:c60f9efd1c36408858d7a90f27ad061313707852b23faa84d02ad526a54fda51
+size 900821

models/vocabulary/crh_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "crh",
-  "vocabulary_size": 51581,
+  "vocabulary_size": 51458,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.1506807050021212,
+    "type_token_ratio": 0.1508716000337303,
     "coverage": {
-      "top_100": 0.4134086438841732,
-      "top_1000": 0.5791304225875555,
-      "top_5000": 0.7107789686755324,
-      "top_10000": 0.7672291584127752
+      "top_100": 0.4143566884351957,
+      "top_1000": 0.5797709619690993,
+      "top_5000": 0.7111434099448135,
+      "top_10000": 0.7674249735310927
     },
-    "hapax_count": 77350,
-    "hapax_ratio": 0.599933297655335,
-    "total_documents": 29389
+    "hapax_count": 77361,
+    "hapax_ratio": 0.6005402929692049,
+    "total_documents": 29399
   }
 }

models/word_markov/crh_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7aa9bdb634e256e31b79d824958c523b31cd09097f73541b17948bf1af86cd78
-size 4147698
+oid sha256:4f4a577ec1eba21b3b29f379827a69e8d4da741390d39a79242a6c4a9d70beb5
+size 4104733

models/word_markov/crh_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "crh",
-  "unique_contexts": 128781,
-  "total_transitions": 826268
+  "unique_contexts": 128666,
+  "total_transitions": 824433
 }

models/word_markov/crh_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4febe8a6192229f809c3e676345cedfae6db5b2e8a8029ea9bd763e24109c749
-size 8051739
+oid sha256:95669d6788baa6c73e434309f34cbba62d64323a3bf17083fc73406e88936c24
+size 8017265

models/word_markov/crh_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "crh",
-  "unique_contexts": 384958,
-  "total_transitions": 796879
+  "unique_contexts": 383467,
+  "total_transitions": 795034
 }