Upload all models and assets for bs (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: bs
-language_name: BS
+language_name: Bosnian
 language_family: slavic_south
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-slavic_south
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.707
+    value: 4.709
   - name: best_isotropy
     type: isotropy
-    value: 0.6837
+    value: 0.6791
   - name: vocabulary_size
     type: vocab
     value: 0
-generated: 2026-01-03
+generated: 2026-01-04
 ---
 
-# BS - Wikilangs Models
+# Bosnian - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **BS** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Bosnian** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.624x | 3.62 | 0.1219% | 1,310,495 |
-| **16k** | 4.031x | 4.03 | 0.1355% | 1,178,214 |
-| **32k** | 4.403x | 4.40 | 0.1481% | 1,078,528 |
-| **64k** | 4.707x 🏆 | 4.71 | 0.1583% | 1,008,868 |
+| **8k** | 3.626x | 3.63 | 0.1221% | 1,306,515 |
+| **16k** | 4.032x | 4.03 | 0.1358% | 1,174,869 |
+| **32k** | 4.404x | 4.40 | 0.1483% | 1,075,596 |
+| **64k** | 4.709x 🏆 | 4.71 | 0.1586% | 1,005,898 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Marija Amalija Austrijska se može odnositi na: Mariju Amaliju Josipu Anu caricu ...`
+**Sample 1:** `Vrpolje Ljubomir je naseljeno mjesto u gradu Trebinju, Bosna i Hercegovina. Stan...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁marija ▁ama lija ▁austri jska ▁se ▁može ▁odnositi ▁na : ... (+24 more)` | 34 |
-| 16k | `▁marija ▁ama lija ▁austrijska ▁se ▁može ▁odnositi ▁na : ▁mari ... (+21 more)` | 31 |
-| 32k | `▁marija ▁ama lija ▁austrijska ▁se ▁može ▁odnositi ▁na : ▁mariju ... (+19 more)` | 29 |
-| 64k | `▁marija ▁ama lija ▁austrijska ▁se ▁može ▁odnositi ▁na : ▁mariju ... (+16 more)` | 26 |
+| 8k | `▁vr polje ▁lju bo mir ▁je ▁naseljeno ▁mjesto ▁u ▁gradu ... (+16 more)` | 26 |
+| 16k | `▁vr polje ▁ljubo mir ▁je ▁naseljeno ▁mjesto ▁u ▁gradu ▁trebinju ... (+13 more)` | 23 |
+| 32k | `▁vr polje ▁ljubomir ▁je ▁naseljeno ▁mjesto ▁u ▁gradu ▁trebinju , ... (+12 more)` | 22 |
+| 64k | `▁vrpolje ▁ljubomir ▁je ▁naseljeno ▁mjesto ▁u ▁gradu ▁trebinju , ▁bosna ... (+11 more)` | 21 |
 
-**Sample 2:** `Margetići su naseljeno mjesto u općini Novi Travnik, Bosna i Hercegovina. Stanov...`
+**Sample 2:** `Kobatovci su naseljeno mjesto u gradu Laktaši, Bosna i Hercegovina. Stanovništvo...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁mar ge ti ći ▁su ▁naseljeno ▁mjesto ▁u ▁općini ▁novi ... (+15 more)` | 25 |
-| 16k | `▁mar ge tići ▁su ▁naseljeno ▁mjesto ▁u ▁općini ▁novi ▁travnik ... (+14 more)` | 24 |
-| 32k | `▁mar ge tići ▁su ▁naseljeno ▁mjesto ▁u ▁općini ▁novi ▁travnik ... (+14 more)` | 24 |
-| 64k | `▁marge tići ▁su ▁naseljeno ▁mjesto ▁u ▁općini ▁novi ▁travnik , ... (+13 more)` | 23 |
+| 8k | `▁ko ba to vci ▁su ▁naseljeno ▁mjesto ▁u ▁gradu ▁la ... (+17 more)` | 27 |
+| 16k | `▁koba to vci ▁su ▁naseljeno ▁mjesto ▁u ▁gradu ▁lakta ši ... (+14 more)` | 24 |
+| 32k | `▁koba tovci ▁su ▁naseljeno ▁mjesto ▁u ▁gradu ▁laktaši , ▁bosna ... (+11 more)` | 21 |
+| 64k | `▁koba tovci ▁su ▁naseljeno ▁mjesto ▁u ▁gradu ▁laktaši , ▁bosna ... (+11 more)` | 21 |
 
-**Sample 3:** `Lilić je naseljeno mjesto u općini Srbac, Bosna i Hercegovina. Stanovništvo Refe...`
+**Sample 3:** `Decenija 780-ih trajala je od 1. januara 780. do 31. decembra 789. godine. Događ...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁li lić ▁je ▁naseljeno ▁mjesto ▁u ▁općini ▁sr bac , ... (+13 more)` | 23 |
-| 16k | `▁li lić ▁je ▁naseljeno ▁mjesto ▁u ▁općini ▁sr bac , ... (+13 more)` | 23 |
-| 32k | `▁li lić ▁je ▁naseljeno ▁mjesto ▁u ▁općini ▁srbac , ▁bosna ... (+11 more)` | 21 |
-| 64k | `▁li lić ▁je ▁naseljeno ▁mjesto ▁u ▁općini ▁srbac , ▁bosna ... (+11 more)` | 21 |
+| 8k | `▁dece nija ▁ 7 8 0 - ih ▁traja la ... (+31 more)` | 41 |
+| 16k | `▁decenija ▁ 7 8 0 - ih ▁trajala ▁je ▁od ... (+29 more)` | 39 |
+| 32k | `▁decenija ▁ 7 8 0 - ih ▁trajala ▁je ▁od ... (+29 more)` | 39 |
+| 64k | `▁decenija ▁ 7 8 0 - ih ▁trajala ▁je ▁od ... (+29 more)` | 39 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.707x compression
-- **Lowest UNK Rate:** 8k with 0.1219% unknown tokens
+- **Best Compression:** 64k achieves 4.709x compression
+- **Lowest UNK Rate:** 8k with 0.1221% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 79,811 | 16.28 | 657,818 | 9.9% | 28.8% |
-| **2-gram** | Subword | 329 🏆 | 8.36 | 10,932 | 62.1% | 98.9% |
-| **3-gram** | Word | 98,469 | 16.59 | 914,717 | 11.8% | 30.2% |
-| **3-gram** | Subword | 3,226 | 11.66 | 100,952 | 20.8% | 64.4% |
-| **4-gram** | Word | 131,360 | 17.00 | 1,461,546 | 13.0% | 31.0% |
-| **4-gram** | Subword | 21,068 | 14.36 | 689,562 | 8.6% | 31.6% |
+| **2-gram** | Word | 80,810 | 16.30 | 664,455 | 9.9% | 28.7% |
+| **2-gram** | Subword | 328 🏆 | 8.36 | 10,943 | 62.1% | 98.9% |
+| **3-gram** | Word | 100,258 | 16.61 | 924,847 | 11.7% | 30.0% |
+| **3-gram** | Subword | 3,216 | 11.65 | 100,916 | 20.8% | 64.5% |
+| **4-gram** | Word | 134,611 | 17.04 | 1,482,132 | 12.9% | 30.8% |
+| **4-gram** | Subword | 20,996 | 14.36 | 689,460 | 8.6% | 31.6% |
+| **5-gram** | Word | 88,861 | 16.44 | 1,107,611 | 15.0% | 34.2% |
+| **5-gram** | Subword | 89,572 | 16.45 | 2,357,541 | 4.7% | 18.4% |
 
 ### Top 5 N-grams by Size
 
@@ -150,21 +162,21 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `spiralna galaksija` | 91,081 |
-| 2 | `vanjski linkovi` | 67,671 |
-| 3 | `se u` | 45,349 |
-| 4 | `reference vanjski` | 43,829 |
+| 1 | `spiralna galaksija` | 91,078 |
+| 2 | `vanjski linkovi` | 68,061 |
+| 3 | `se u` | 45,470 |
+| 4 | `reference vanjski` | 44,256 |
 | 5 | `ngc ic` | 40,015 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `reference vanjski linkovi` | 43,767 |
-| 2 | `prečkasta spiralna galaksija` | 32,672 |
-| 3 | `zavod za statistiku` | 22,677 |
-| 4 | `popisu stanovništva godine` | 20,724 |
-| 5 | `na popisu stanovništva` | 20,183 |
+| 1 | `reference vanjski linkovi` | 44,193 |
+| 2 | `prečkasta spiralna galaksija` | 32,671 |
+| 3 | `zavod za statistiku` | 22,679 |
+| 4 | `popisu stanovništva godine` | 20,723 |
+| 5 | `na popisu stanovništva` | 20,184 |
 
 **4-grams (Word):**
 
@@ -173,45 +185,65 @@ Below are sample sentences tokenized with each vocabulary size:
 | 1 | `na popisu stanovništva godine` | 20,088 |
 | 2 | `državni zavod za statistiku` | 14,619 |
 | 3 | `broj stanovnika po popisima` | 13,853 |
-| 4 | `reference vanjski linkovi u` | 13,661 |
-| 5 | `pogledajte novi opći katalog` | 13,518 |
+| 4 | `reference vanjski linkovi u` | 13,677 |
+| 5 | `novi opći katalog spisak` | 13,518 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `također pogledajte novi opći katalog` | 13,518 |
+| 2 | `pogledajte novi opći katalog spisak` | 13,517 |
+| 3 | `historija do teritorijalne reorganizacije u` | 13,436 |
+| 4 | `interaktivni ngc online katalog astronomska` | 13,248 |
+| 5 | `ngc online katalog astronomska baza` | 13,248 |
 
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `a _` | 5,676,715 |
-| 2 | `e _` | 4,422,458 |
-| 3 | `j e` | 3,860,834 |
-| 4 | `i _` | 3,755,142 |
-| 5 | `_ s` | 3,354,838 |
+| 1 | `a _` | 5,724,674 |
+| 2 | `e _` | 4,473,918 |
+| 3 | `j e` | 3,904,782 |
+| 4 | `i _` | 3,802,145 |
+| 5 | `_ s` | 3,388,803 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `j e _` | 1,718,703 |
-| 2 | `n a _` | 1,228,627 |
-| 3 | `_ n a` | 1,166,020 |
-| 4 | `_ j e` | 1,117,037 |
-| 5 | `_ p o` | 1,073,431 |
+| 1 | `j e _` | 1,738,823 |
+| 2 | `n a _` | 1,237,973 |
+| 3 | `_ n a` | 1,177,081 |
+| 4 | `_ j e` | 1,128,189 |
+| 5 | `_ p o` | 1,086,240 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `_ j e _` | 915,938 |
-| 2 | `i j a _` | 454,224 |
-| 3 | `_ n a _` | 449,657 |
-| 4 | `_ s e _` | 393,812 |
-| 5 | `i j e _` | 313,056 |
+| 1 | `_ j e _` | 924,709 |
+| 2 | `i j a _` | 457,403 |
+| 3 | `_ n a _` | 454,266 |
+| 4 | `_ s e _` | 399,769 |
+| 5 | `i j e _` | 316,944 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `a _ j e _` | 263,188 |
+| 2 | `_ g o d i` | 195,374 |
+| 3 | `g o d i n` | 192,967 |
+| 4 | `o _ j e _` | 190,942 |
+| 5 | `_ n g c _` | 158,105 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram (subword) with 329
+- **Best Perplexity:** 2-gram (subword) with 328
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~32% of corpus
+- **Coverage:** Top-1000 patterns cover ~18% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.9822 | 1.975 | 9.95 | 1,092,943 | 1.8% |
-| **1** | Subword | 1.0154 | 2.021 | 7.75 | 3,822 | 0.0% |
-| **2** | Word | 0.3063 | 1.237 | 1.90 | 10,856,748 | 69.4% |
-| **2** | Subword | 0.9503 | 1.932 | 6.62 | 29,614 | 5.0% |
-| **3** | Word | 0.1026 | 1.074 | 1.20 | 20,575,067 | 89.7% |
-| **3** | Subword | 0.9528 | 1.936 | 5.48 | 195,969 | 4.7% |
-| **4** | Word | 0.0377 🏆 | 1.026 | 1.06 | 24,704,289 | 96.2% |
-| **4** | Subword | 0.9418 | 1.921 | 4.19 | 1,073,568 | 5.8% |
+| **1** | Word | 0.9835 | 1.977 | 9.99 | 1,096,434 | 1.7% |
+| **1** | Subword | 1.0155 | 2.022 | 7.71 | 3,863 | 0.0% |
+| **2** | Word | 0.3071 | 1.237 | 1.90 | 10,934,441 | 69.3% |
+| **2** | Subword | 0.9460 | 1.927 | 6.59 | 29,789 | 5.4% |
+| **3** | Word | 0.1029 | 1.074 | 1.20 | 20,758,711 | 89.7% |
+| **3** | Subword | 0.9514 | 1.934 | 5.47 | 196,125 | 4.9% |
+| **4** | Word | 0.0378 🏆 | 1.027 | 1.06 | 24,939,260 | 96.2% |
+| **4** | Subword | 0.9416 | 1.921 | 4.19 | 1,073,504 | 5.8% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `i iskazivano pod bizantijsku teritoriju poljski 2 kola za zvornik i hercegovine a nastavljaju sa dru...`
-2. `je reference vanjski linkovi www portalanalitika me 31 okt 3 kalisz david friedrich ehrendorfer ehre...`
-3. `u plazmi parcijalni derivati otkriveni napad na evropskom prvenstvu su klasifikovani kao tipično do ...`
+1. `i sfrj popis ostali su nove ere ce espanyol olímpic lluís d očigledno drevni grad u`
+2. `je počeo zanimati za testiranje je holoenzim počinje u genima patofiziološki mehanizam samouništenja...`
+3. `u zemaljskom muzeju i rukama do teritorijalne reorganizacije u 13 33 923 0 plesni parovi još`
 
 **Context Size 2:**
 
-1. `spiralna galaksija s0 a ic 0 66 spiralna galaksija s također pogledajte orah čvor`
-2. `vanjski linkovi na hromosomu 13 proteini sindrom`
-3. `se u gorskom kotaru velebitu lici i član glavnog odbora stranka je bila 1 nositeljica u 1`
+1. `spiralna galaksija s ic 0 51 nepoznato 3 0 3 uglovnih minuta s a d p gdje`
+2. `vanjski linkovi ic ic na aladin pregledaču ic katalog na ngc ic objekti sljedeći spisak sadrži deset`
+3. `se u četvrtfinale potom je bila poljska glumica koja iza sebe thomasa morgensterna koch vor morgenst...`
 
 **Context Size 3:**
 
-1. `reference vanjski linkovi skakaonica paul ausserleitner izgrađena je u periodu od do godine je tu i ...`
-2. `prečkasta spiralna galaksija koja je udaljena oko 162 miliona sg od zemlje i nalazi se u sazviježđu ...`
-3. `zavod za statistiku republike hrvatske reference vanjski linkovi u sloveniji u primorsko notranjskoj...`
+1. `reference vanjski linkovi zvanični sajt općine teslić`
+2. `prečkasta spiralna galaksija sbab p ngc 5 41 emisijska maglina en također pogledajte novi opći katal...`
+3. `zavod za statistiku i evidenciju fnrj i sfrj popis stanovništva i godine knjiga narodnosni i vjerski...`
 
 **Context Size 4:**
 
-1. `na popisu stanovništva godine črešnjevec je imao 19 stanovnika broj stanovnika po popisima 553 492 5...`
-2. `državni zavod za statistiku naselja i stanovništvo republike hrvatske 118 128 172 210 219 245 266 26...`
-3. `broj stanovnika po popisima 89 120 123 reference vanjski linkovi u sloveniji u posavskoj regiji hist...`
+1. `na popisu stanovništva godine naseljeno mjesto majkovi je imalo 273 stanovnika broj stanovnika po po...`
+2. `državni zavod za statistiku naselja i stanovništvo republike hrvatske 23 0 84 85 129 118 110 149 130...`
+3. `broj stanovnika po popisima 31 38 napomena u nastalo izdvajanjem dijela iz naselja buk vlaka i opuze...`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `_ebiću_hakorona_`
-2. `araćiskano_nit_l`
-3. `i_d_portajemojse`
+1. `_diintk,_d,_pri_`
+2. `arafužde_0452)_b`
+3. `inavjuc_stodite_`
 
 **Context Size 2:**
 
-1. `a_augućina_bi_med`
-2. `e_vrhipa,_ceaka_s`
-3. `jeglazrobelikimal`
+1. `a_stal)_teiftupng`
+2. `e_podilnetskimost`
+3. `jedin_štvoji_izvi`
 
 **Context Size 3:**
 
-1. `je_poznac_ženerga_`
-2. `na_galaksija_sa_ce`
-3. `_najblijača_objavl`
+1. `je_nazi_se_daklene`
+2. `na_predočan_heime_`
+3. `_nama_prija,_datim`
 
 **Context Size 4:**
 
-1. `_je_u_složenja_dell`
-2. `ija_roadbez_von_lew`
-3. `_na_pozici_bosnu!_t`
+1. `_je_od_na_15_462_sb`
+2. `ija_deset_na_od_tri`
+3. `_na_prema_oltara_ko`
 
 
 ### Key Findings
 
 - **Best Predictability:** Context-4 (word) with 96.2% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (1,073,568 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (1,073,504 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,48 +346,48 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 502,911 |
-| Total Tokens | 32,206,003 |
-| Mean Frequency | 64.04 |
+| Vocabulary Size | 504,813 |
+| Total Tokens | 32,497,466 |
+| Mean Frequency | 64.38 |
 | Median Frequency | 4 |
-| Frequency Std Dev | 2755.29 |
+| Frequency Std Dev | 2777.29 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | i | 934,658 |
-| 2 | je | 922,929 |
-| 3 | u | 915,148 |
-| 4 | na | 453,346 |
-| 5 | se | 397,234 |
-| 6 | su | 288,366 |
-| 7 | od | 268,408 |
-| 8 | za | 263,873 |
-| 9 | 1 | 253,982 |
-| 10 | ngc | 206,398 |
+| 1 | i | 945,166 |
+| 2 | je | 931,753 |
+| 3 | u | 924,423 |
+| 4 | na | 457,967 |
+| 5 | se | 403,233 |
+| 6 | su | 292,637 |
+| 7 | od | 271,227 |
+| 8 | za | 266,768 |
+| 9 | 1 | 253,853 |
+| 10 | ngc | 206,389 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | polikristale | 2 |
-| 2 | ṁ | 2 |
-| 3 | bikristal | 2 |
-| 4 | polikristal | 2 |
-| 5 | misesov | 2 |
-| 6 | abstractmethod | 2 |
-| 7 | ugođen | 2 |
-| 8 | unifilarni | 2 |
-| 9 | neomurani | 2 |
-| 10 | arhebakterije | 2 |
+| 1 | antiinfektivne | 2 |
+| 2 | veditors | 2 |
+| 3 | esac | 2 |
+| 4 | martirosyan | 2 |
+| 5 | neuzimanje | 2 |
+| 6 | spekarski | 2 |
+| 7 | probabilizamski | 2 |
+| 8 | dtl | 2 |
+| 9 | setap | 2 |
+| 10 | visoravani | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 0.9663 |
-| R² (Goodness of Fit) | 0.999465 |
+| Zipf Coefficient | 0.9660 |
+| R² (Goodness of Fit) | 0.999467 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
@@ -363,15 +395,15 @@ Below are text samples generated from each subword-based Markov chain model:
 | Top N Words | Coverage |
 |-------------|----------|
 | Top 100 | 32.1% |
-| Top 1,000 | 53.2% |
-| Top 5,000 | 68.8% |
+| Top 1,000 | 53.1% |
+| Top 5,000 | 68.7% |
 | Top 10,000 | 75.7% |
 
 ### Key Findings
 
 - **Zipf Compliance:** R²=0.9995 indicates excellent adherence to Zipf's law
 - **High Frequency Dominance:** Top 100 words cover 32.1% of corpus
-- **Long Tail:** 492,911 words needed for remaining 24.3% coverage
+- **Long Tail:** 494,813 words needed for remaining 24.3% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.6837 🏆 | 0.3607 | N/A | N/A |
-| **mono_64d** | 64 | 0.6836 | 0.2874 | N/A | N/A |
-| **mono_128d** | 128 | 0.6518 | 0.2275 | N/A | N/A |
+| **mono_32d** | 32 | 0.6791 🏆 | 0.3557 | N/A | N/A |
+| **mono_64d** | 64 | 0.6789 | 0.2931 | N/A | N/A |
+| **mono_128d** | 128 | 0.6505 | 0.2294 | N/A | N/A |
+| **aligned_32d** | 32 | 0.6791 | 0.3517 | 0.1940 | 0.5160 |
+| **aligned_64d** | 64 | 0.6789 | 0.2923 | 0.3680 | 0.7380 |
+| **aligned_128d** | 128 | 0.6505 | 0.2262 | 0.4520 | 0.7800 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.6837 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.2919. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** mono_32d with 0.6791 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.2914. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 45.2% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **0.860** | High formulaic/idiomatic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -426,20 +461,20 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Prefixes
 | Prefix | Examples |
 |--------|----------|
-| `-pr` | pričvršćenom, prihova, prethodnica |
-| `-po` | pokusu, povljana, polimerskim |
+| `-pr` | promotriti, pristrasno, priznavajući |
+| `-po` | podstilova, postporođajno, položene |
 
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-a` | kolomina, puferska, prihova |
-| `-e` | ostatke, akademske, dominantnotype |
-| `-i` | ristovski, ukrajinski, dovođeni |
-| `-om` | pričvršćenom, nekontrolisanom, polupustinjskom |
-| `-na` | kolomina, povljana, financirana |
-| `-ja` | ašiklija, preimenovanja, grobalja |
-| `-ma` | polusestrama, metaloenzima, falconsima |
-| `-im` | tvrtkovim, polimerskim, briljantnim |
+| `-a` | ćamila, afrića, canaima |
+| `-e` | candace, emilie, feničane |
+| `-i` | izrađujući, promotriti, opstruktivni |
+| `-om` | holivudskom, ekvatorom, mckaganom |
+| `-na` | odoljena, zloćudna, interamericana |
+| `-ni` | opstruktivni, bogobojazni, normani |
+| `-og` | vazdušnog, nanizanog, modularnog |
+| `-ja` | inkrustacija, gaskonja, bradikardija |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -447,18 +482,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `kovi` | 1.59x | 619 contexts | kovin, kovič, ković |
-| `anov` | 1.57x | 625 contexts | hanov, kanov, banov |
-| `selj` | 2.07x | 82 contexts | seljo, selja, seljak |
-| `alak` | 2.52x | 33 contexts | malak, talak, stalak |
-| `vanj` | 1.71x | 170 contexts | vanje, kvanj, vanja |
-| `renc` | 1.98x | 75 contexts | renci, renco, renca |
-| `acij` | 1.55x | 220 contexts | lacij, acije, acija |
-| `alna` | 1.97x | 58 contexts | šalna, malna, valna |
-| `jekt` | 1.82x | 78 contexts | objekt, subjekt, trajekt |
-| `iral` | 1.50x | 164 contexts | miral, ziral, viral |
-| `njsk` | 1.56x | 134 contexts | vnjski, anjski, vanjsk |
-| `egov` | 1.55x | 114 contexts | negov, begov, begovo |
+| `anov` | 1.53x | 627 contexts | panov, šanov, anova |
+| `ijsk` | 1.54x | 411 contexts | ijski, šijska, azijske |
+| `renc` | 2.13x | 74 contexts | renca, renci, renco |
+| `kovi` | 1.39x | 620 contexts | okovi, ković, kovič |
+| `alak` | 2.51x | 33 contexts | malak, talak, malaku |
+| `selj` | 1.97x | 81 contexts | selja, seljo, crselj |
+| `jekt` | 1.94x | 77 contexts | objekt, subjekt, objektu |
+| `iral` | 1.65x | 165 contexts | viral, ziral, miral |
+| `ksij` | 2.04x | 55 contexts | iksija, oleksij, taksiju |
+| `vanj` | 1.56x | 169 contexts | vanju, vanji, kvanj |
+| `acij` | 1.45x | 219 contexts | acije, acija, lacij |
+| `bjek` | 2.29x | 27 contexts | ribjek, žabjek, objeki |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -466,16 +501,16 @@ This table shows which prefixes and suffixes most frequently co-occur on the sam
 
 | Prefix | Suffix | Frequency | Examples |
 |--------|--------|-----------|----------|
-| `-pr` | `-a` | 63 words | prementuma, protivpožarna |
-| `-po` | `-a` | 56 words | potomcima, pobunila |
-| `-pr` | `-i` | 49 words | pravokutni, propuštajući |
-| `-pr` | `-e` | 49 words | primijećuje, prirasle |
-| `-po` | `-i` | 48 words | poručivši, poliribosomi |
-| `-po` | `-e` | 32 words | poupée, popularizacije |
-| `-po` | `-ma` | 12 words | potomcima, poslodavcima |
-| `-pr` | `-om` | 12 words | preporodnom, preuranjenom |
-| `-pr` | `-ja` | 12 words | protozoologija, pribavlja |
-| `-pr` | `-na` | 11 words | protivpožarna, protimozina |
+| `-pr` | `-a` | 64 words | pripaja, prezentska |
+| `-po` | `-a` | 56 words | posttestikulska, pokroviteljima |
+| `-pr` | `-e` | 50 words | prijestupne, pregljeve |
+| `-pr` | `-i` | 45 words | prevareni, prebacivani |
+| `-po` | `-e` | 39 words | potterove, polusušne |
+| `-po` | `-i` | 36 words | populaciji, potterovi |
+| `-pr` | `-om` | 14 words | pramajkom, prustom |
+| `-pr` | `-na` | 14 words | pravougaona, pretražena |
+| `-pr` | `-ni` | 12 words | prevareni, prebacivani |
+| `-po` | `-na` | 11 words | ponosna, polipropilena |
 
 ### 6.5 Recursive Morpheme Segmentation
 
@@ -483,26 +518,28 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
-| umoljanima | **`umol-ja-ni-ma`** | 7.5 | `umol` |
-| melkumani | **`melku-ma-ni`** | 6.0 | `melku` |
-| postepenim | **`po-stepen-im`** | 6.0 | `stepen` |
-| skečevima | **`skečevi-ma`** | 4.5 | `skečevi` |
-| nesigurnostima | **`nesigurnosti-ma`** | 4.5 | `nesigurnosti` |
-| balansiranje | **`balansiran-je`** | 4.5 | `balansiran` |
-| ignoriranje | **`ignoriran-je`** | 4.5 | `ignoriran` |
-| pobjesnio | **`po-bjesnio`** | 4.5 | `bjesnio` |
-| integrirani | **`integrira-ni`** | 4.5 | `integrira` |
-| rutherfordovom | **`rutherfordov-om`** | 4.5 | `rutherfordov` |
-| karlingom | **`karling-om`** | 4.5 | `karling` |
-| kriopirinom | **`kriopirin-om`** | 4.5 | `kriopirin` |
-| šezdesetim | **`šezdeset-im`** | 4.5 | `šezdeset` |
-| pojašnjena | **`po-jašn-je-na`** | 4.5 | `jašn` |
-| akreditiranje | **`akreditiran-je`** | 4.5 | `akreditiran` |
+| nerazvijenog | **`nerazvijen-og`** | 4.5 | `nerazvijen` |
+| langleyja | **`langley-ja`** | 4.5 | `langley` |
+| nadvratnikom | **`nadvratnik-om`** | 4.5 | `nadvratnik` |
+| zahvaćenog | **`zahvaćen-og`** | 4.5 | `zahvaćen` |
+| posigurno | **`po-sigurno`** | 4.5 | `sigurno` |
+| nepostojanja | **`nepostojan-ja`** | 4.5 | `nepostojan` |
+| dramatizirana | **`dramatizira-na`** | 4.5 | `dramatizira` |
+| newtonovom | **`newtonov-om`** | 4.5 | `newtonov` |
+| bertoluccija | **`bertolucci-ja`** | 4.5 | `bertolucci` |
+| uravnoteženog | **`uravnotežen-og`** | 4.5 | `uravnotežen` |
+| ilustriranom | **`ilustriran-om`** | 4.5 | `ilustriran` |
+| saobraćajne | **`saobraćaj-ne`** | 4.5 | `saobraćaj` |
+| herlihyja | **`herlihy-ja`** | 4.5 | `herlihy` |
+| čehovljevog | **`čehovljev-og`** | 4.5 | `čehovljev` |
+| rječnikom | **`rječnik-om`** | 4.5 | `rječnik` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language BS appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Bosnian shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
+
+> **Note on Idiomaticity:** The high Idiomaticity Gap suggests a large number of frequent multi-word expressions or formulaic sequences that are statistically distinct from their component parts.
 
 ---
 ## 7. Summary & Recommendations
@@ -514,7 +551,7 @@ The language BS appears to be more isolating or has a highly fixed vocabulary. W
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
 | Tokenizer | **64k BPE** | Best compression (4.71x) |
-| N-gram | **2-gram** | Lowest perplexity (329) |
+| N-gram | **2-gram** | Lowest perplexity (328) |
 | Markov | **Context-4** | Highest predictability (96.2%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
@@ -729,4 +766,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 10:03:26*
+*Report Date: 2026-01-04 01:24:53*

models/embeddings/aligned/bs_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:740829966f542e125e0a6b95c1b7da610ab7c35d274dc35d57f43ab6e485f4d7
+size 1383848108

models/embeddings/aligned/bs_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "bs", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bs_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e1ca3f4a7c3a558b4af8b49b44215f677b05de9fed1d880d0d3fb9ba66dd7c6
+size 65664

models/embeddings/aligned/bs_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bs",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 75890,
+  "vocab_size": 345296
+}

models/embeddings/aligned/bs_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea51e943e3c6ea30e248d9460b6c4d1b6559605686fe024d5b33bb35504b7392
+size 350660780

models/embeddings/aligned/bs_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "bs", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bs_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0276d485aaee612fa243a827e739b82210e174fee894a60aa32a46a5ff02e581
+size 4224

models/embeddings/aligned/bs_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bs",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 75890,
+  "vocab_size": 345296
+}

models/embeddings/aligned/bs_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3495bb7aa5110757d145b1b199a1927442dcd7a16b1c358b1402697d9d04db8c
+size 695056556

models/embeddings/aligned/bs_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "bs", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bs_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f21c0408f6d7bc24d580c2c77eb5b2900b8cf68af1146f96d946c1fb6261262b
+size 16512

models/embeddings/aligned/bs_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bs",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 75890,
+  "vocab_size": 345296
+}

models/embeddings/monolingual/bs_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:150a739475c8b2802aaaa35679a6ec5946f23a4489669763729d419b2cf329bf
-size 1381600692
+oid sha256:740829966f542e125e0a6b95c1b7da610ab7c35d274dc35d57f43ab6e485f4d7
+size 1383848108

models/embeddings/monolingual/bs_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 343142
+  "vocab_size": 345296
 }

models/embeddings/monolingual/bs_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5db060f6dd5915bae4076dfd0dea4755e1f4664360e6713434947ebf5dd49b94
-size 350067636
+oid sha256:ea51e943e3c6ea30e248d9460b6c4d1b6559605686fe024d5b33bb35504b7392
+size 350660780

models/embeddings/monolingual/bs_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 343142
+  "vocab_size": 345296
 }

models/embeddings/monolingual/bs_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09f0ccc9cb7ef218444a015a72c973a1aa64b7424c24eac2b2b5f999e7bae986
-size 693911988
+oid sha256:3495bb7aa5110757d145b1b199a1927442dcd7a16b1c358b1402697d9d04db8c
+size 695056556

models/embeddings/monolingual/bs_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 343142
+  "vocab_size": 345296
 }

models/subword_markov/bs_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c413639e86aff70a1e366f239702d90f57805452729b30f47b449e1b50e9d0d1
-size 236628
+oid sha256:580ef87b49130ed49ff159c8164d31b6f01d98433a64c9d538e7f41b624f15ee
+size 236177

models/subword_markov/bs_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "bs",
-  "unique_contexts": 3822,
-  "total_transitions": 217153057
+  "unique_contexts": 3863,
+  "total_transitions": 219211726
 }

models/subword_markov/bs_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07af575a720ce0333065d1365fd6f0dc1e9c196dbbdf47dae5915e384670002e
-size 1556026
+oid sha256:0b44918ca92271d7ed19465fa1de0e2edf8270a015094764f1dd3af1a02912d2
+size 1542690

models/subword_markov/bs_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "bs",
-  "unique_contexts": 29614,
-  "total_transitions": 217057401
+  "unique_contexts": 29789,
+  "total_transitions": 219115700
 }

models/subword_markov/bs_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8734e9ff242be1673110fbadecec4921d18b749fab506afd58084c6454f10179
-size 8759226
+oid sha256:a0d2df121b534fe36e2676f6be2cef5bcc80ea59efedb4bd2118d50e988d8fee
+size 8745505

models/subword_markov/bs_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "bs",
-  "unique_contexts": 195969,
-  "total_transitions": 216961745
+  "unique_contexts": 196125,
+  "total_transitions": 219019674
 }

models/subword_markov/bs_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fcd4e07fd4b57bbf32facc8ddf98017e371bfdad9f2a2932def87de4cad2a336
-size 32234978
+oid sha256:00be54a57894e61db8fbe018a657c97967bd1b3028837f33b29e619a61e89ee7
+size 32299886

models/subword_markov/bs_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "bs",
-  "unique_contexts": 1073568,
-  "total_transitions": 216866089
+  "unique_contexts": 1073504,
+  "total_transitions": 218923648
 }

models/subword_ngram/bs_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5088ff625c4275eba000b58efc27a957d0b2fe00df41f79272702d5740b6aa5a
-size 149061
+oid sha256:fee76f4adf84ca3f5114d7be749b77ab81ed9f3041c91b8c0888caceb5928db2
+size 149172

models/subword_ngram/bs_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "bs",
-  "unique_ngrams": 10932,
-  "total_ngrams": 217153057
+  "unique_ngrams": 10943,
+  "total_ngrams": 219211726
 }

models/subword_ngram/bs_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:994dcf177eaabe0675b5f4a1b329598035e90a3b5e97cc7f73d373979f3cdd23
-size 1254949
+oid sha256:53af4f27c5189ebee07d3fe789367fcd9ed5c57bf7aeff40da8109f67d307d89
+size 1247121

models/subword_ngram/bs_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "bs",
-  "unique_ngrams": 100952,
-  "total_ngrams": 217057401
+  "unique_ngrams": 100916,
+  "total_ngrams": 219115700
 }

models/subword_ngram/bs_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f9e9075117ccaa25ec707915fefe1acd5957f11ad9c94308474e7238513466d0
-size 7917058
+oid sha256:493b3aeb57e4418945f830de6391962f50b3deb0d23b8bdd85e6dd73162c7f94
+size 7925488

models/subword_ngram/bs_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "bs",
-  "unique_ngrams": 689562,
-  "total_ngrams": 216961745
+  "unique_ngrams": 689460,
+  "total_ngrams": 219019674
 }

models/subword_ngram/bs_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae7d1c19cd8eb86bddb97090cabdd03fb51af53fb565cdad10f3bff2d9b1a76b
+size 27632630

models/subword_ngram/bs_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "bs",
+  "unique_ngrams": 2357541,
+  "total_ngrams": 218923648
+}

models/tokenizer/bs_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6009fbe251dc69f2b102003863ae8d942c6cdb255917c51ca2f5748f2025e0b0
-size 511740
+oid sha256:2a42f1e4f0c742af636bc4f9cd1fc5c10c74811ad70a6170ee378d34a7b6fd67
+size 511742

models/tokenizer/bs_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f77069e783a7216648e25a6ebc87d0c8bc974ab6c4df1509692cd565a485b703
-size 795090
+oid sha256:35d176df4774b50024cf589aa5da550821b6335b5301795e55e94ff2bd5f4773
+size 795058

models/tokenizer/bs_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b340f78c1c205596e8e78b4b37abd3a0af06a9dcb987769156d87e8ae5bc64f
-size 1373883
+oid sha256:684435959a3684e964ce32b638f6a396b529fc7c0ece3c9149dd917afdea92c6
+size 1374870

models/tokenizer/bs_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a827c60064fd24d771a22fb0ed577a444662b37250d01ef94952af5e5f0ea59f
-size 374247
+oid sha256:8c5449a33c22c6608ed944aecfee971b503f45a932a12b8c92959c404fe89e15
+size 374178

models/vocabulary/bs_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49eafd3b4bbbba80874eda2ec39ac7b2e78de842040a668fddae30d8d872a494
-size 7547245
+oid sha256:15ba8661b0cf334bc60af9d9ed01ee1e509947a45de434528dfcf5e273de9124
+size 7568773

models/vocabulary/bs_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "bs",
-  "vocabulary_size": 502911,
+  "vocabulary_size": 504813,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.033338017781857625,
+    "type_token_ratio": 0.03314747772147322,
     "coverage": {
-      "top_100": 0.3148584842736538,
-      "top_1000": 0.5222226568897175,
-      "top_5000": 0.6752045594187739,
-      "top_10000": 0.7431974443827949
+      "top_100": 0.31476820159732916,
+      "top_1000": 0.5218994335542112,
+      "top_5000": 0.6749973088128356,
+      "top_10000": 0.7430279817583144
     },
-    "hapax_count": 590458,
-    "hapax_ratio": 0.5400354317709758,
-    "total_documents": 95656
+    "hapax_count": 592020,
+    "hapax_ratio": 0.5397540008369551,
+    "total_documents": 96026
   }
 }

models/word_markov/bs_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec8d96e1bb6fe5416cce99b5c6d8559355a902c39366ee4bac65c211144112c5
-size 99684168
+oid sha256:1b893d21739c73d36f14e45cbde52362688c17b34d7fd32d0cca267604158044
+size 100155421

models/word_markov/bs_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "bs",
-  "unique_contexts": 1092943,
-  "total_transitions": 32700805
+  "unique_contexts": 1096434,
+  "total_transitions": 32993460
 }

models/word_markov/bs_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b7fcee8bc4b37f470ffa1f616fa833e1a90112ecac1f412ad0a81c5ac29b58b
-size 277373494
+oid sha256:76a70b558427179c36b025fd7b4a7d09b9a87688e4f48193eb7d46ffa81b6ae9
+size 279585529

models/word_markov/bs_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "bs",
-  "unique_contexts": 10856748,
-  "total_transitions": 32605149
+  "unique_contexts": 10934441,
+  "total_transitions": 32897434
 }