Upload all models and assets for av (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: av
-language_name: AV
+language_name: Avar
 language_family: caucasian_northeast
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-caucasian_northeast
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.697
+    value: 4.685
   - name: best_isotropy
     type: isotropy
-    value: 0.8716
+    value: 0.8604
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# AV - Wikilangs Models
+# Avar - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **AV** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Avar** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.636x | 3.64 | 0.0717% | 252,363 |
-| **16k** | 4.040x | 4.04 | 0.0797% | 227,147 |
-| **32k** | 4.391x | 4.40 | 0.0866% | 208,961 |
-| **64k** | 4.697x 🏆 | 4.70 | 0.0927% | 195,348 |
+| **8k** | 3.628x | 3.63 | 0.0828% | 245,293 |
+| **16k** | 4.030x | 4.03 | 0.0919% | 220,825 |
+| **32k** | 4.383x | 4.39 | 0.1000% | 203,018 |
+| **64k** | 4.685x 🏆 | 4.69 | 0.1069% | 189,944 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Хъипчахъ () гъорлъе уна жибго Хъипчахъ росу. Гьеб росулъ гьабула . росаби`
+**Sample 1:** `19-абилеб Октябр — грегорианияб календаралда рекъон къо (високоснияб соналъ — св...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁хъ ип ч ахъ ▁() ▁гъорлъе ▁уна ▁жибго ▁хъ ип ... (+9 more)` | 19 |
-| 16k | `▁хъ ип ч ахъ ▁() ▁гъорлъе ▁уна ▁жибго ▁хъ ип ... (+9 more)` | 19 |
-| 32k | `▁хъипчахъ ▁() ▁гъорлъе ▁уна ▁жибго ▁хъипчахъ ▁росу . ▁гьеб ▁росулъ ... (+3 more)` | 13 |
-| 64k | `▁хъипчахъ ▁() ▁гъорлъе ▁уна ▁жибго ▁хъипчахъ ▁росу . ▁гьеб ▁росулъ ... (+3 more)` | 13 |
+| 8k | `▁ 1 9 - абилеб ▁октябр ▁— ▁грегорианияб ▁календаралда ▁рекъон ... (+18 more)` | 28 |
+| 16k | `▁ 1 9 - абилеб ▁октябр ▁— ▁грегорианияб ▁календаралда ▁рекъон ... (+18 more)` | 28 |
+| 32k | `▁ 1 9 - абилеб ▁октябр ▁— ▁грегорианияб ▁календаралда ▁рекъон ... (+18 more)` | 28 |
+| 64k | `▁ 1 9 - абилеб ▁октябр ▁— ▁грегорианияб ▁календаралда ▁рекъон ... (+18 more)` | 28 |
 
-**Sample 2:** `26-абилеб июль — грегорианияб календаралда рекъон къо (високоснияб соналъ — свер...`
+**Sample 2:** `Пинкь яги ГьанамагӀ (латиназул мацӀалда bulla; Bullae) — гӀадамасул лага-черх. л...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁ 2 6 - абилеб ▁июль ▁— ▁грегорианияб ▁календаралда ▁рекъон ... (+19 more)` | 29 |
-| 16k | `▁ 2 6 - абилеб ▁июль ▁— ▁грегорианияб ▁календаралда ▁рекъон ... (+19 more)` | 29 |
-| 32k | `▁ 2 6 - абилеб ▁июль ▁— ▁грегорианияб ▁календаралда ▁рекъон ... (+19 more)` | 29 |
-| 64k | `▁ 2 6 - абилеб ▁июль ▁— ▁грегорианияб ▁календаралда ▁рекъон ... (+19 more)` | 29 |
+| 8k | `▁п ин кь ▁яги ▁гьан ам агӏ ▁( латиназул ▁мацӏалда ... (+18 more)` | 28 |
+| 16k | `▁пин кь ▁яги ▁гьан амагӏ ▁( латиназул ▁мацӏалда ▁b ul ... (+15 more)` | 25 |
+| 32k | `▁пин кь ▁яги ▁гьан амагӏ ▁( латиназул ▁мацӏалда ▁b ul ... (+14 more)` | 24 |
+| 64k | `▁пинкь ▁яги ▁гьанамагӏ ▁( латиназул ▁мацӏалда ▁b ul la ; ... (+11 more)` | 21 |
 
-**Sample 3:** `() ккола Билкан районалда гъорлъе унеб росу. росаби`
+**Sample 3:** `22-абилеб Октябр — грегорианияб календаралда рекъон къо (високоснияб соналъ — св...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁() ▁ккола ▁билкан ▁районалда ▁гъорлъе ▁унеб ▁росу . ▁росаби` | 9 |
-| 16k | `▁() ▁ккола ▁билкан ▁районалда ▁гъорлъе ▁унеб ▁росу . ▁росаби` | 9 |
-| 32k | `▁() ▁ккола ▁билкан ▁районалда ▁гъорлъе ▁унеб ▁росу . ▁росаби` | 9 |
-| 64k | `▁() ▁ккола ▁билкан ▁районалда ▁гъорлъе ▁унеб ▁росу . ▁росаби` | 9 |
+| 8k | `▁ 2 2 - абилеб ▁октябр ▁— ▁грегорианияб ▁календаралда ▁рекъон ... (+18 more)` | 28 |
+| 16k | `▁ 2 2 - абилеб ▁октябр ▁— ▁грегорианияб ▁календаралда ▁рекъон ... (+18 more)` | 28 |
+| 32k | `▁ 2 2 - абилеб ▁октябр ▁— ▁грегорианияб ▁календаралда ▁рекъон ... (+18 more)` | 28 |
+| 64k | `▁ 2 2 - абилеб ▁октябр ▁— ▁грегорианияб ▁календаралда ▁рекъон ... (+18 more)` | 28 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.697x compression
-- **Lowest UNK Rate:** 8k with 0.0717% unknown tokens
+- **Best Compression:** 64k achieves 4.685x compression
+- **Lowest UNK Rate:** 8k with 0.0828% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 3,247 | 11.66 | 6,413 | 22.5% | 54.2% |
-| **2-gram** | Subword | 428 🏆 | 8.74 | 4,133 | 57.8% | 96.7% |
-| **3-gram** | Word | 2,834 | 11.47 | 6,427 | 26.0% | 57.0% |
-| **3-gram** | Subword | 3,424 | 11.74 | 28,949 | 23.6% | 62.9% |
-| **4-gram** | Word | 8,629 | 13.07 | 17,392 | 16.9% | 37.2% |
-| **4-gram** | Subword | 15,875 | 13.95 | 119,337 | 12.4% | 36.8% |
+| **2-gram** | Word | 3,089 | 11.59 | 6,523 | 23.7% | 56.2% |
+| **2-gram** | Subword | 424 🏆 | 8.73 | 4,120 | 58.0% | 96.7% |
+| **3-gram** | Word | 2,775 | 11.44 | 6,745 | 26.4% | 58.9% |
+| **3-gram** | Subword | 3,361 | 11.71 | 28,903 | 23.9% | 63.4% |
+| **4-gram** | Word | 8,260 | 13.01 | 18,126 | 17.8% | 39.8% |
+| **4-gram** | Subword | 15,393 | 13.91 | 119,191 | 12.7% | 37.5% |
+| **5-gram** | Word | 7,813 | 12.93 | 15,673 | 16.8% | 39.4% |
+| **5-gram** | Subword | 38,531 | 15.23 | 222,134 | 8.4% | 26.5% |
 
 ### Top 5 N-grams by Size
 
@@ -150,68 +162,88 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `росу буго` | 509 |
-| 2 | `лъугьа бахъинал` | 496 |
-| 3 | `география росу` | 461 |
-| 4 | `цо цо` | 455 |
-| 5 | `of the` | 441 |
+| 1 | `росу буго` | 710 |
+| 2 | `география росу` | 660 |
+| 3 | `мухъалъул росаби` | 578 |
+| 4 | `буго мухъалъул` | 530 |
+| 5 | `мухъалъул росу` | 523 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `география росу буго` | 448 |
-| 2 | `лъугьа бахъинал гьаруна` | 368 |
-| 3 | `бахъинал гьаруна хвана` | 358 |
-| 4 | `байрамал лъугьа бахъинал` | 353 |
-| 5 | `гьаруна хвана ишараби` | 352 |
+| 1 | `география росу буго` | 645 |
+| 2 | `росу буго мухъалъул` | 523 |
+| 3 | `лъугьа бахъинал гьаруна` | 368 |
+| 4 | `бахъинал гьаруна хвана` | 358 |
+| 5 | `байрамал лъугьа бахъинал` | 353 |
 
 **4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `лъугьа бахъинал гьаруна х��ана` | 358 |
-| 2 | `байрамал лъугьа бахъинал гьаруна` | 352 |
-| 3 | `къо байрамал лъугьа бахъинал` | 351 |
-| 4 | `бахъинал гьаруна хвана ишараби` | 349 |
-| 5 | `демография ккола моноэтникияб авар` | 329 |
+| 1 | `география росу буго мухъалъул` | 513 |
+| 2 | `лъугьа бахъинал гьаруна хвана` | 358 |
+| 3 | `байрамал лъугьа бахъинал гьаруна` | 352 |
+| 4 | `къо байрамал лъугьа бахъинал` | 351 |
+| 5 | `бахъинал гьаруна хвана ишараби` | 349 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `къо байрамал лъугьа бахъинал гьаруна` | 350 |
+| 2 | `лъугьа бахъинал гьаруна хвана ишараби` | 349 |
+| 3 | `байрамал лъугьа бахъинал гьаруна хвана` | 348 |
+| 4 | `демография ккола моноэтникияб авар росулъун` | 305 |
+| 5 | `география росу буго мухъалъул марказ` | 279 |
 
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `а л` | 82,724 |
-| 2 | `л _` | 63,062 |
-| 3 | `л ъ` | 52,236 |
-| 4 | `а _` | 52,185 |
-| 5 | `у л` | 49,900 |
+| 1 | `а л` | 85,368 |
+| 2 | `л _` | 64,955 |
+| 3 | `л ъ` | 53,561 |
+| 4 | `а _` | 52,853 |
+| 5 | `у л` | 50,828 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `у л _` | 33,240 |
-| 2 | `л ъ у` | 30,603 |
-| 3 | `ъ у л` | 25,387 |
-| 4 | `а л ъ` | 23,574 |
-| 5 | `_ г ь` | 22,295 |
+| 1 | `у л _` | 34,266 |
+| 2 | `л ъ у` | 31,682 |
+| 3 | `ъ у л` | 26,429 |
+| 4 | `а л ъ` | 24,583 |
+| 5 | `_ г ь` | 22,014 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `л ъ у л` | 23,988 |
-| 2 | `ъ у л _` | 21,518 |
-| 3 | `а л ъ у` | 16,083 |
-| 4 | `а л д а` | 11,383 |
-| 5 | `_ г ь е` | 11,094 |
+| 1 | `л ъ у л` | 25,035 |
+| 2 | `ъ у л _` | 22,571 |
+| 3 | `а л ъ у` | 16,980 |
+| 4 | `а л д а` | 11,684 |
+| 5 | `_ г ь е` | 10,931 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `л ъ у л _` | 22,224 |
+| 2 | `а л ъ у л` | 15,591 |
+| 3 | `я л ъ у л` | 7,776 |
+| 4 | `а л д а _` | 7,381 |
+| 5 | `_ б у г о` | 5,843 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram (subword) with 428
+- **Best Perplexity:** 2-gram (subword) with 424
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~37% of corpus
+- **Coverage:** Top-1000 patterns cover ~26% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.6602 | 1.580 | 3.57 | 91,234 | 34.0% |
-| **1** | Subword | 1.1781 | 2.263 | 9.32 | 1,145 | 0.0% |
-| **2** | Word | 0.1256 | 1.091 | 1.21 | 324,656 | 87.4% |
-| **2** | Subword | 0.9990 | 1.999 | 5.68 | 10,664 | 0.1% |
-| **3** | Word | 0.0281 | 1.020 | 1.04 | 392,645 | 97.2% |
-| **3** | Subword | 0.7935 | 1.733 | 3.66 | 60,534 | 20.6% |
-| **4** | Word | 0.0114 🏆 | 1.008 | 1.02 | 406,500 | 98.9% |
-| **4** | Subword | 0.5614 | 1.476 | 2.33 | 221,628 | 43.9% |
+| **1** | Word | 0.6594 | 1.579 | 3.57 | 90,954 | 34.1% |
+| **1** | Subword | 1.1677 | 2.247 | 9.26 | 1,148 | 0.0% |
+| **2** | Word | 0.1264 | 1.092 | 1.22 | 323,475 | 87.4% |
+| **2** | Subword | 0.9998 | 2.000 | 5.69 | 10,625 | 0.0% |
+| **3** | Word | 0.0288 | 1.020 | 1.04 | 392,122 | 97.1% |
+| **3** | Subword | 0.7938 | 1.734 | 3.67 | 60,414 | 20.6% |
+| **4** | Word | 0.0121 🏆 | 1.008 | 1.02 | 406,770 | 98.8% |
+| **4** | Subword | 0.5607 | 1.475 | 2.33 | 221,366 | 43.9% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `ва бищун хирияб рокьуе рецц гьабун росулӏ историкияб кьучӏги x гіасру ккола гъуниб округалъул цӏигӏу...`
-2. `буго гьединго гьолокьги бекьизабун буго шумеразул ги къасимехалъ бачӏингун чагӏазда макьаби контрола...`
-3. `ккола гьижрияб соналъул 29 август цояб гьелъул буго гьанже батизе бегьула 1 гуржистаналъул бищун бор...`
+1. `ва испан фонология цогидал туркиял мацӏаз чанго шагьрияб гӏумру ��шавалда хурхарал феодализм социум с...`
+2. `буго республикалъул рутул мухъ буго шартіияб рикікіеналдалъун гьабураб бищун це б грузинский алфавит...`
+3. `бугеб муниципалияб гӏуцӏи гъорлӏе рачуна чӏужуялда хурхарал цогидал киналго хвана ишараби мугъчӏваял...`
 
 **Context Size 2:**
 
-1. `росу буго лъарагӏлъиялда хасавхъала мухъалда хасавхъалаялдаса 24 км ялъ жанубиябгин бакъбаккудехун а...`
-2. `лъугьа бахъинал гьаруна хвана ишараби мугъчӏваял гь балагье хіужаби иццал адабият гіурус маціалда бу...`
-3. `география росу буго лъарагӏлъиялда дибирилросу мухъалда дибирилросуялдаса 10 км ялъ шималиябгин бакъ...`
+1. `росу буго мухъалъул марказ лъаратӏаса 22 км лъ жанубияб бакъбаккудехун ралъдал гьурматӏаса 968 метра...`
+2. `география росу буго мухъалъул марказ лъаратӏаса 0 5 41 9 12 гуржиял 617 401 253 10 0`
+3. `буго мухъалъул центер уркарахъалдаса бакътӏерхьудехун демография референсал мухъалъул росаби мухъ ро...`
 
 **Context Size 3:**
 
-1. `география росу буго мухъалъул марказ хӏебдаса 15 километралъ бакъбаккудехун халкъ мугъчӏваял регӏела...`
-2. `лъугьа бахъинал гьаруна хвана ишараби мугъчӏваял гь балагье трактат адабият тайпаби изданиял`
-3. `байрамал лъугьа бахъинал гьаруна хвана ишараби мугъчӏваял гь балагье трактат адабият тайпаби издания...`
+1. `география росу буго мухъалъул марказ лъаратӏаса 22 км алъ демография ккола моноэтникияб авар росулъу...`
+2. `росу буго мухъалъул центер уркарахъалдаса жанубияб бакътӏерхьудехун ралъдал гьурматӏаса борхалъи буг...`
+3. `лъугьа бахъинал гьаруна хвана ишараби мугъчӏваял гь балагье трактат адабият тайпаби изданиял`
 
 **Context Size 4:**
 
-1. `байрамал лъугьа бахъинал гьаруна хвана ишараби мугъчӏваял гь балагье`
-2. `къо байрамал лъугьа бахъинал гьаруна хвана ишараби мугъчӏваял гь балагье`
-3. `лъугьа бахъинал гьаруна хвана ишараби мугъчӏваял гь балагье`
+1. `география росу буго мухъалъул марказ лъаратӏаса 5 км алъ шималалиябгин бакъбаккудехун аваргӏоралъул ...`
+2. `байрамал лъугьа бахъинал гьаруна хвана ишараби мугъчӏваял гь балагье`
+3. `къо байрамал лъугьа бахъинал гьаруна хвана ишараби мугъчӏваял гь балагье`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `_iv–_гъугіо_usth`
-2. `абиза_и._д_2%_ке`
-3. `л_—_ilissoldan_|`
+1. `_ссва_—_1_вадаре`
+2. `ан._ия_в._тӏавар`
+3. `лдацӏиялъухъуск;`
 
 **Context Size 2:**
 
-1. `алъахъану_кконие_`
-2. `л_адекалабаяракӏ)`
-3. `лъул_кіаялъул_на_`
+1. `алдастияб_6_киябр`
+2. `л_джибацӏаниякеап`
+3. `лъул_бакъго_рахъе`
 
 **Context Size 3:**
 
-1. `ул_къотӏагораний_в`
-2. `лъул_реал_карт_гӏа`
-3. `ъулгун_ар-рип_хъал`
+1. `ул_намен_гьеб_раса`
+2. `лъулго_справенция)`
+3. `ъул_яги_перации_«г`
 
 **Context Size 4:**
 
-1. `лъул_хіалалда_чӏали`
-2. `ъул_большая_и_казбе`
-3. `алъул_руго_9:_мугъч`
+1. `лъул_ассив_гьел_ккв`
+2. `ъул_ківар_география`
+3. `алъулалде._борхалъу`
 
 
 ### Key Findings
 
-- **Best Predictability:** Context-4 (word) with 98.9% predictability
+- **Best Predictability:** Context-4 (word) with 98.8% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (221,628 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (221,366 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,64 +346,64 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 34,392 |
-| Total Tokens | 405,867 |
-| Mean Frequency | 11.80 |
+| Vocabulary Size | 34,315 |
+| Total Tokens | 413,611 |
+| Mean Frequency | 12.05 |
 | Median Frequency | 3 |
-| Frequency Std Dev | 73.46 |
+| Frequency Std Dev | 77.17 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | ва | 7,245 |
-| 2 | буго | 5,074 |
-| 3 | ккола | 2,830 |
-| 4 | бугеб | 2,699 |
-| 5 | гьеб | 2,222 |
-| 6 | росу | 2,175 |
-| 7 | мухъалъул | 2,030 |
-| 8 | цо | 1,833 |
-| 9 | the | 1,815 |
-| 10 | соналъ | 1,799 |
+| 1 | ва | 7,138 |
+| 2 | буго | 5,684 |
+| 3 | бугеб | 2,903 |
+| 4 | ккола | 2,872 |
+| 5 | росу | 2,838 |
+| 6 | мухъалъул | 2,671 |
+| 7 | гьеб | 2,178 |
+| 8 | росдал | 1,902 |
+| 9 | the | 1,812 |
+| 10 | цо | 1,800 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | долтул | 2 |
-| 2 | кӏалалдаса | 2 |
-| 3 | шаргі | 2 |
-| 4 | харитӏун | 2 |
-| 5 | луткунги | 2 |
-| 6 | беглъуда | 2 |
-| 7 | къацӏар | 2 |
-| 8 | мичегь | 2 |
-| 9 | хъурукал | 2 |
-| 10 | мягьле | 2 |
+| 1 | уркутамахьи | 2 |
+| 2 | континуумалде | 2 |
+| 3 | къулецӏмаги | 2 |
+| 4 | гьаркӏасуниб | 2 |
+| 5 | махӏарги | 2 |
+| 6 | пилибхиталъул | 2 |
+| 7 | заповедникалда | 2 |
+| 8 | пилибхит | 2 |
+| 9 | лъалъадул | 2 |
+| 10 | хӏанчӏи | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 0.9506 |
-| R² (Goodness of Fit) | 0.993368 |
+| Zipf Coefficient | 0.9572 |
+| R² (Goodness of Fit) | 0.993745 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 22.5% |
-| Top 1,000 | 50.8% |
-| Top 5,000 | 73.6% |
-| Top 10,000 | 83.3% |
+| Top 100 | 23.1% |
+| Top 1,000 | 51.6% |
+| Top 5,000 | 74.2% |
+| Top 10,000 | 83.6% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9934 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 22.5% of corpus
-- **Long Tail:** 24,392 words needed for remaining 16.7% coverage
+- **Zipf Compliance:** R²=0.9937 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 23.1% of corpus
+- **Long Tail:** 24,315 words needed for remaining 16.4% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.8716 🏆 | 0.3278 | N/A | N/A |
-| **mono_64d** | 64 | 0.7240 | 0.2821 | N/A | N/A |
-| **mono_128d** | 128 | 0.2461 | 0.2702 | N/A | N/A |
+| **mono_32d** | 32 | 0.8604 | 0.3207 | N/A | N/A |
+| **mono_64d** | 64 | 0.7367 | 0.2711 | N/A | N/A |
+| **mono_128d** | 128 | 0.2721 | 0.2530 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8604 🏆 | 0.3335 | 0.0200 | 0.1400 |
+| **aligned_64d** | 64 | 0.7367 | 0.2791 | 0.0280 | 0.1780 |
+| **aligned_128d** | 128 | 0.2721 | 0.2649 | 0.0820 | 0.2540 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.8716 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.2934. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** aligned_32d with 0.8604 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.2870. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 8.2% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **0.488** | High formulaic/idiomatic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -426,21 +461,19 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Prefixes
 | Prefix | Examples |
 |--------|----------|
-| `-гь` | гьамчукъотӏи, гьечӏони, гьаркьалги |
-| `-гӏ` | гӏасру, гӏаракъи, гӏелмуялде |
-| `-ма` | материялъул, машгьадалда, магіарухъ |
+| `-ба` | батӏалъуда, батӏа, бахъинаро |
 
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-л` | сабабал, материялъул, рикӏкӏиналъул |
-| `-а` | елена, современника, шагьаралда |
-| `-ул` | материялъул, рикӏкӏиналъул, хӏажиевасул |
-| `-да` | шагьаралда, машгьадалда, флорида |
-| `-ъул` | материялъул, рикӏкӏиналъул, медициналъул |
-| `-лъул` | материялъул, рикӏкӏиналъул, медициналъул |
-| `-ал` | сабабал, кьурахарал, къезавидал |
-| `-лда` | шагьаралда, машгьадалда, борталда |
+| `-л` | субтропикиял, кумикал, риччалел |
+| `-а` | лъараца, тіалъиялда, анатолиялдаса |
+| `-ул` | агьлулъиялъул, кипралъул, урарталъул |
+| `-ъул` | агьлулъиялъул, кипралъул, урарталъул |
+| `-лъул` | агьлулъиялъул, кипралъул, урарталъул |
+| `-да` | тіалъиялда, текстазда, батӏалъуда |
+| `-ал` | кумикал, туарегал, я́сал |
+| `-ги` | тахшагьарлъунги, яги, фортисги |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -448,18 +481,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `алъу` | 1.82x | 100 contexts | алъул, далъун, ралъуе |
-| `агьа` | 1.89x | 59 contexts | дагьа, багьа, загьаб |
-| `ялъу` | 2.04x | 43 contexts | ялъул, аялъул, ялъуни |
-| `ьабу` | 2.16x | 29 contexts | гьабу, гьабун, кьабун |
-| `иялъ` | 1.96x | 36 contexts | абиялъе, химиялъ, лъиялъе |
-| `иялд` | 1.83x | 35 contexts | сиялда, азиялда, азиялде |
-| `анал` | 1.42x | 70 contexts | данал, канал, ханал |
-| `ралъ` | 1.49x | 53 contexts | ралъад, ралъуе, хералъ |
-| `буге` | 2.00x | 17 contexts | бугел, бугез, бугеб |
-| `иста` | 2.02x | 16 contexts | систан, христа, лазистан |
-| `лдас` | 2.06x | 15 contexts | лдаса, алдаса, ялдаса |
-| `азда` | 1.62x | 32 contexts | мазда, раздан, ишазда |
+| `алъу` | 1.88x | 101 contexts | алъул, далъун, малъун |
+| `ялъу` | 2.05x | 41 contexts | ялъул, ялъуни, аялъул |
+| `ьабу` | 2.11x | 29 contexts | гьабу, гьабун, кьабун |
+| `агьа` | 1.75x | 59 contexts | багьа, дагьа, шагьав |
+| `иялъ` | 1.85x | 36 contexts | химиялъ, биялъул, армиялъ |
+| `анал` | 1.48x | 70 contexts | канал, ханал, данал |
+| `иялд` | 1.69x | 36 contexts | сиялда, азиялде, азиялда |
+| `огра` | 1.87x | 22 contexts | географ, фотограф, этнограф |
+| `азда` | 1.67x | 31 contexts | гьазда, ишазда, раздан |
+| `налд` | 1.64x | 31 contexts | иналда, доналд, иналде |
+| `гъор` | 2.15x | 13 contexts | гъорлі, гъорлъ, гъорлӏ |
+| `лдас` | 2.01x | 15 contexts | лдаса, ялдаса, алдаса |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -467,16 +500,16 @@ This table shows which prefixes and suffixes most frequently co-occur on the sam
 
 | Prefix | Suffix | Frequency | Examples |
 |--------|--------|-----------|----------|
-| `-гь` | `-л` | 44 words | гьавамухъал, гьудулзабазул |
-| `-ма` | `-л` | 40 words | мажлисалъул, маринил |
-| `-гӏ` | `-л` | 38 words | гӏурусазул, гӏалиевалъул |
-| `-ма` | `-а` | 35 words | макъалоялда, малъана |
-| `-гӏ` | `-а` | 29 words | гӏуцӏиялда, гӏодула |
-| `-гь` | `-а` | 28 words | гьада, гьала |
-| `-гӏ` | `-ул` | 25 words | гӏурусазул, гӏалиевалъул |
-| `-гь` | `-ул` | 24 words | гьудулзабазул, гьезул |
-| `-ма` | `-ул` | 21 words | мажлисалъул, мактабалъул |
-| `-ма` | `-да` | 16 words | макъалоялда, макъалаялда |
+| `-ба` | `-л` | 36 words | багьадурасул, бакътӏерхьул |
+| `-ба` | `-а` | 34 words | багъа, батӏалъана |
+| `-ба` | `-ул` | 17 words | багьадурасул, бакътӏерхьул |
+| `-ба` | `-ун` | 16 words | бахчун, бахъбаккудехун |
+| `-ба` | `-да` | 16 words | бащалъуда, балазда |
+| `-ба` | `-ал` | 11 words | бахӏсал, бакъбаккулал |
+| `-ба` | `-ъул` | 8 words | бавариялъул, баталйоналъул |
+| `-ба` | `-лда` | 8 words | бахъиялда, бахшалда |
+| `-ба` | `-ги` | 6 words | бакӏалъулги, бахӏарзабиги |
+| `-ба` | `-лъул` | 6 words | бавариялъул, баталйоналъул |
 
 ### 6.5 Recursive Morpheme Segmentation
 
@@ -484,26 +517,28 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
-| руччабаздаги | **`руччабаз-да-ги`** | 6.0 | `руччабаз` |
-| хронологиялъул | **`хронология-лъул`** | 4.5 | `хронология` |
-| теориялда | **`теория-лда`** | 4.5 | `теория` |
-| къавмазул | **`къавмаз-ул`** | 4.5 | `къавмаз` |
-| къанагӏатал | **`къанагӏат-ал`** | 4.5 | `къанагӏат` |
-| групалъул | **`група-лъул`** | 4.5 | `група` |
-| ракьалъул | **`ракьа-лъул`** | 4.5 | `ракьа` |
-| такрарлъул | **`такрар-лъул`** | 4.5 | `такрар` |
-| алвеолариялги | **`алвеолариял-ги`** | 4.5 | `алвеолариял` |
-| европалъул | **`европа-лъул`** | 4.5 | `европа` |
-| гьабулаго | **`гь-абулаго`** | 4.5 | `абулаго` |
-| рахъалъги | **`рахъалъ-ги`** | 4.5 | `рахъалъ` |
-| пассажирги | **`пассажир-ги`** | 4.5 | `пассажир` |
-| партиялъул | **`партия-лъул`** | 4.5 | `партия` |
-| оппозициялъул | **`оппозиция-лъул`** | 4.5 | `оппозиция` |
+| къуръаналги | **`къуръан-ал-ги`** | 6.0 | `къуръан` |
+| ханасдаги | **`ханас-да-ги`** | 6.0 | `ханас` |
+| элементалги | **`элемент-ал-ги`** | 6.0 | `элемент` |
+| гьелъулги | **`гьел-ъул-ги`** | 6.0 | `гьел` |
+| гьармониялда | **`гьармония-лда`** | 4.5 | `гьармония` |
+| гьолокьги | **`гьолокь-ги`** | 4.5 | `гьолокь` |
+| хьондасебги | **`хьондасеб-ги`** | 4.5 | `хьондасеб` |
+| районазул | **`районаз-ул`** | 4.5 | `районаз` |
+| аскаразда | **`аскараз-да`** | 4.5 | `аскараз` |
+| экономикаги | **`экономика-ги`** | 4.5 | `экономика` |
+| процессазул | **`процессаз-ул`** | 4.5 | `процессаз` |
+| насрудиницаги | **`насрудиница-ги`** | 4.5 | `насрудиница` |
+| бугиланги | **`бугилан-ги`** | 4.5 | `бугилан` |
+| рагьаразул | **`рагьараз-ул`** | 4.5 | `рагьараз` |
+| минскалъул | **`минска-лъул`** | 4.5 | `минска` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language AV appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Avar shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
+
+> **Note on Idiomaticity:** The high Idiomaticity Gap suggests a large number of frequent multi-word expressions or formulaic sequences that are statistically distinct from their component parts.
 
 ---
 ## 7. Summary & Recommendations
@@ -514,9 +549,9 @@ The language AV appears to be more isolating or has a highly fixed vocabulary. W
 
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
-| Tokenizer | **64k BPE** | Best compression (4.70x) |
-| N-gram | **2-gram** | Lowest perplexity (428) |
-| Markov | **Context-4** | Highest predictability (98.9%) |
+| Tokenizer | **64k BPE** | Best compression (4.69x) |
+| N-gram | **2-gram** | Lowest perplexity (424) |
+| Markov | **Context-4** | Highest predictability (98.8%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
 
@@ -730,4 +765,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 05:23:28*
+*Report Date: 2026-01-03 18:29:30*

models/embeddings/aligned/av_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52180d54d8a01ed8dc3b1895710307bdf710be093a465547504c72098c065d36
+size 1036200777

models/embeddings/aligned/av_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "av", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/av_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af5e663dfe219030fa1896110d65b3a61da08e57239e14eb6c20a27e254e50bd
+size 65664

models/embeddings/aligned/av_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "av",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 1918,
+  "vocab_size": 11646
+}

models/embeddings/aligned/av_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8251e7217d1b123fe6bbdd58c049f85f4fc0cb3e8fbb502325d478f85fd750a6
+size 259256649

models/embeddings/aligned/av_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "av", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/av_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82c4c5c405118ebedb62e164f95bf087157687447049e734ffa92982f10b987b
+size 4224

models/embeddings/aligned/av_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "av",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 1918,
+  "vocab_size": 11646
+}

models/embeddings/aligned/av_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ecf1ac33cdc6b30c9842ac01a360012c70da565e13e7a104b878aa86e312449
+size 518238025

models/embeddings/aligned/av_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "av", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/av_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2010d3e67a96f1d9214b69d0ddea791f43f4604b27e2a6ccf657a97032095dd6
+size 16512

models/embeddings/aligned/av_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "av",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 1918,
+  "vocab_size": 11646
+}

models/embeddings/monolingual/av_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:086998639c3328d1f88a224eb653bef49f8aa011b5880d7feb85792cbe742361
-size 1036208926
+oid sha256:52180d54d8a01ed8dc3b1895710307bdf710be093a465547504c72098c065d36
+size 1036200777

models/embeddings/monolingual/av_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 11654
+  "vocab_size": 11646
 }

models/embeddings/monolingual/av_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6290250d27a46d72b90ede94292690c45a0a3f14bdbf05ab7aa5d07aa2093541
-size 259258654
+oid sha256:8251e7217d1b123fe6bbdd58c049f85f4fc0cb3e8fbb502325d478f85fd750a6
+size 259256649

models/embeddings/monolingual/av_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 11654
+  "vocab_size": 11646
 }

models/embeddings/monolingual/av_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b709ff729efabdd809f80746b620dba08607a5b3377d6237ee6f2434e1eb3c2
-size 518242078
+oid sha256:9ecf1ac33cdc6b30c9842ac01a360012c70da565e13e7a104b878aa86e312449
+size 518238025

models/embeddings/monolingual/av_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 11654
+  "vocab_size": 11646
 }

models/subword_markov/av_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4517ac9631ea8dbfee38f5e0a123dfdfec510e3a9c1219d351bc1fd509b60c17
-size 81084
+oid sha256:38a5fc9cc98043e5e42a3d1b11dc89ca11fc329935cadbb52961882b9b7b83e9
+size 81204

models/subword_markov/av_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "av",
-  "unique_contexts": 1145,
-  "total_transitions": 3671343
+  "unique_contexts": 1148,
+  "total_transitions": 3736506
 }

models/subword_markov/av_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2851c37f629eaaf5084dca27968d6cf91fd8843aa330d489a4440f208d3e60cd
-size 486043
+oid sha256:c720711e9e69392708fe5e71b1b2a63522d60a2455f116510f96f263cce0bf14
+size 478235

models/subword_markov/av_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "av",
-  "unique_contexts": 10664,
-  "total_transitions": 3667770
+  "unique_contexts": 10625,
+  "total_transitions": 3732688
 }

models/subword_markov/av_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e93984ebab9588d00463857f800a79e61319598925a11d5785c24e04809162f3
-size 1681476
+oid sha256:5e23c4390424fb194a144d49a7a4d106ee4815414d1f02b382d16cdce335c3a2
+size 1619923

models/subword_markov/av_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "av",
-  "unique_contexts": 60534,
-  "total_transitions": 3664197
+  "unique_contexts": 60414,
+  "total_transitions": 3728870
 }

models/subword_markov/av_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df78890cb55850cde1e4fbcdb06710bcbf3c0a04113c2fb5a81c53eb29e9a8bc
-size 4643917
+oid sha256:307469cf2a68ecb4dce27966c3ba2bb8cf0af5f6fdd08b5f1f21c93064675ab3
+size 4652863

models/subword_markov/av_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "av",
-  "unique_contexts": 221628,
-  "total_transitions": 3660624
+  "unique_contexts": 221366,
+  "total_transitions": 3725052
 }

models/subword_ngram/av_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:933b0de1b63cb20550e97fa2419bacf817141c74827bc4e101e5d76c7780509c
-size 54611
+oid sha256:878073a5d2c637b1f85fafae2ba4be9157d6c82edb5088bf8dd9cb33ab1a58c8
+size 54488

models/subword_ngram/av_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "av",
-  "unique_ngrams": 4133,
-  "total_ngrams": 3671343
+  "unique_ngrams": 4120,
+  "total_ngrams": 3736506
 }

models/subword_ngram/av_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca32b51e264e1da8fb8ad605682f565e4a0fd97590d147b080946a6cf8993da2
-size 370254
+oid sha256:d6bc323576e241439ae55b2f57ce8af013b971d5e98dff0e748e2504fa6374f4
+size 371643

models/subword_ngram/av_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "av",
-  "unique_ngrams": 28949,
-  "total_ngrams": 3667770
+  "unique_ngrams": 28903,
+  "total_ngrams": 3732688
 }

models/subword_ngram/av_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1511e4aa7f03fe2637243fa72bb121ccae146a0024ee7a81f233a708cc2778bb
-size 1469426
+oid sha256:a16219f855b243bc4265361726c9322bd40b68c3f3ec8c492a2fe9d88d9b9c54
+size 1466503

models/subword_ngram/av_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "av",
-  "unique_ngrams": 119337,
-  "total_ngrams": 3664197
+  "unique_ngrams": 119191,
+  "total_ngrams": 3728870
 }

models/subword_ngram/av_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56df9057219471fe915c809035a735c463eaa460b99a4d6af6abdbb9b6618763
+size 2908784

models/subword_ngram/av_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "av",
+  "unique_ngrams": 222134,
+  "total_ngrams": 3725052
+}

models/tokenizer/av_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95f05447a801c104acaac9e4a45ef1ade9a750529b7be0b69f4ee702b40bdd0f
-size 579855
+oid sha256:c084d64ab2c2f58d5bc9f1f2dc70eace9aeadaeadfd45a28841cae6bf2ef9e53
+size 581313

models/tokenizer/av_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bcc76ebd732e7de0cc3f1d1a7445d6e9812fc5861347bfbdb764a34ab378093a
-size 943739
+oid sha256:c6ff9e1bc49cd0cb18d9360212ae9e8cfe73a0a21e867e0912b1c87f6ad7b19a
+size 946330

models/tokenizer/av_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e25c37c305a3f71acc7fde1f68cea5d1b999e9cc76e1a9ae7e946535b348e7eb
-size 1709450
+oid sha256:dfa371e8e0d4e42c77bb7b2966ee37072f3b5cd2c0535d51bb585bd288e7eb4d
+size 1710003

models/tokenizer/av_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69224dbe8521456dbe0800e5c74893d56ecd3d316ce5785f5ea5922ff2f1af24
-size 404399
+oid sha256:d34afc9d7d1c89ae47b12ffc26333fc4419d35ed30764eaf2e9938e4cce0dc23
+size 405256

models/vocabulary/av_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7310d663e46f21c8df63d5030b41ef164690182783e77aaf70f0bb2efff7bb0
-size 661413
+oid sha256:67880bc9e008144c86d9ad7b24a767201ed2594683e944ac426bacb90f203963
+size 659034

models/vocabulary/av_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "av",
-  "vocabulary_size": 34392,
+  "vocabulary_size": 34315,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.1973921591928009,
+    "type_token_ratio": 0.19363404248081859,
     "coverage": {
-      "top_100": 0.19702269707347111,
-      "top_1000": 0.4456231702442555,
-      "top_5000": 0.6456771851739821,
-      "top_10000": 0.7303661131936867
+      "top_100": 0.20352823373591822,
+      "top_1000": 0.45408257631644405,
+      "top_5000": 0.652138603715743,
+      "top_10000": 0.7354887675205207
     },
-    "hapax_count": 56968,
-    "hapax_ratio": 0.623555166374781,
-    "total_documents": 3573
+    "hapax_count": 56766,
+    "hapax_ratio": 0.6232474390926758,
+    "total_documents": 3818
   }
 }

models/word_markov/av_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e06648f80de88e4993ac542ab602cc9aac6b01b06f56ff96c14658afaf6be279
-size 4391092
+oid sha256:000e753d5507ed19d49b2e77d3c852a9e226efade45d833420c7ab7bff43425b
+size 4379098

models/word_markov/av_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "av",
-  "unique_contexts": 91234,
-  "total_transitions": 459262
+  "unique_contexts": 90954,
+  "total_transitions": 466559
 }

models/word_markov/av_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c68afd2ecd43feece23866755466060024bed9599ffd5b4fb7f0eebba9e44b7c
-size 9606282
+oid sha256:748594fa3dfa2640edc5c4d6d16fa6fc935d127222889082f084b4c57286d0fc
+size 9577859

models/word_markov/av_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "av",
-  "unique_contexts": 324656,
-  "total_transitions": 455689
+  "unique_contexts": 323475,
+  "total_transitions": 462741
 }