Upload all models and assets for br (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: br
-language_name: BR
+language_name: Breton
 language_family: celtic_brythonic
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-celtic_brythonic
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 3.786
+    value: 3.787
   - name: best_isotropy
     type: isotropy
-    value: 0.8171
+    value: 0.8154
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# BR - Wikilangs Models
+# Breton - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **BR** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Breton** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.235x | 3.24 | 0.4490% | 793,680 |
-| **16k** | 3.460x | 3.46 | 0.4803% | 742,059 |
-| **32k** | 3.645x | 3.65 | 0.5060% | 704,331 |
-| **64k** | 3.786x 🏆 | 3.79 | 0.5255% | 678,179 |
+| **8k** | 3.238x | 3.24 | 0.4518% | 788,643 |
+| **16k** | 3.463x | 3.46 | 0.4832% | 737,391 |
+| **32k** | 3.647x | 3.65 | 0.5089% | 700,148 |
+| **64k** | 3.787x 🏆 | 3.79 | 0.5284% | 674,255 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Monsano zo ur gumun italian e proviñs Ancona, er Marche. Marche Proviñs Ancona`
+**Sample 1:** `Concetta Barra a oa ur ganerez hag un aktourez italian ha dreist-holl napolitane...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁mon s ano ▁zo ▁ur ▁gumun ▁italian ▁e ▁proviñs ▁anc ... (+9 more)` | 19 |
-| 16k | `▁mons ano ▁zo ▁ur ▁gumun ▁italian ▁e ▁proviñs ▁ancona , ... (+6 more)` | 16 |
-| 32k | `▁mons ano ▁zo ▁ur ▁gumun ▁italian ▁e ▁proviñs ▁ancona , ... (+6 more)` | 16 |
-| 64k | `▁mons ano ▁zo ▁ur ▁gumun ▁italian ▁e ▁proviñs ▁ancona , ... (+6 more)` | 16 |
+| 8k | `▁conc etta ▁bar ra ▁a ▁oa ▁ur ▁ganerez ▁hag ▁un ... (+30 more)` | 40 |
+| 16k | `▁conc etta ▁barra ▁a ▁oa ▁ur ▁ganerez ▁hag ▁un ▁aktourez ... (+26 more)` | 36 |
+| 32k | `▁conc etta ▁barra ▁a ▁oa ▁ur ▁ganerez ▁hag ▁un ▁aktourez ... (+26 more)` | 36 |
+| 64k | `▁conc etta ▁barra ▁a ▁oa ▁ur ▁ganerez ▁hag ▁un ▁aktourez ... (+22 more)` | 32 |
 
-**Sample 2:** `San Asensio zo ur gumun e proviñs La Rioja en Spagn. Rioja`
+**Sample 2:** `Fénis zo ur gumun italian, e rannvro emren Traoñienn Aosta. Notennoù`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁san ▁as ens io ▁zo ▁ur ▁gumun ▁e ▁proviñs ▁la ... (+5 more)` | 15 |
-| 16k | `▁san ▁as ens io ▁zo ▁ur ▁gumun ▁e ▁proviñs ▁la ... (+5 more)` | 15 |
-| 32k | `▁san ▁as ens io ▁zo ▁ur ▁gumun ▁e ▁proviñs ▁la ... (+5 more)` | 15 |
-| 64k | `▁san ▁as ens io ▁zo ▁ur ▁gumun ▁e ▁proviñs ▁la ... (+5 more)` | 15 |
+| 8k | `▁f én is ▁zo ▁ur ▁gumun ▁italian , ▁e ▁rannvro ... (+6 more)` | 16 |
+| 16k | `▁f én is ▁zo ▁ur ▁gumun ▁italian , ▁e ▁rannvro ... (+5 more)` | 15 |
+| 32k | `▁f én is ▁zo ▁ur ▁gumun ▁italian , ▁e ▁rannvro ... (+5 more)` | 15 |
+| 64k | `▁fén is ▁zo ▁ur ▁gumun ▁italian , ▁e ▁rannvro ▁emren ... (+4 more)` | 14 |
 
-**Sample 3:** `Segusino zo ur gumun e proviñs Treviso e Veneto, en Italia.`
+**Sample 3:** `Cervera del Río Alhama zo ur gumun e kumuniezh emren La Rioja e Spagn.`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁seg us ino ▁zo ▁ur ▁gumun ▁e ▁proviñs ▁trev iso ... (+6 more)` | 16 |
-| 16k | `▁seg us ino ▁zo ▁ur ▁gumun ▁e ▁proviñs ▁treviso ▁e ... (+5 more)` | 15 |
-| 32k | `▁seg us ino ▁zo ▁ur ▁gumun ▁e ▁proviñs ▁treviso ▁e ... (+5 more)` | 15 |
-| 64k | `▁seg us ino ▁zo ▁ur ▁gumun ▁e ▁proviñs ▁treviso ▁e ... (+5 more)` | 15 |
+| 8k | `▁c erv era ▁del ▁río ▁al h ama ▁zo ▁ur ... (+9 more)` | 19 |
+| 16k | `▁cerv era ▁del ▁río ▁al h ama ▁zo ▁ur ▁gumun ... (+8 more)` | 18 |
+| 32k | `▁cerv era ▁del ▁río ▁al h ama ▁zo ▁ur ▁gumun ... (+8 more)` | 18 |
+| 64k | `▁cervera ▁del ▁río ▁alhama ▁zo ▁ur ▁gumun ▁e ▁kumuniezh ▁emren ... (+5 more)` | 15 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 3.786x compression
-- **Lowest UNK Rate:** 8k with 0.4490% unknown tokens
+- **Best Compression:** 64k achieves 3.787x compression
+- **Lowest UNK Rate:** 8k with 0.4518% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 37,349 | 15.19 | 296,192 | 13.7% | 32.0% |
-| **2-gram** | Subword | 294 🏆 | 8.20 | 11,776 | 65.3% | 98.9% |
-| **3-gram** | Word | 128,487 | 16.97 | 570,380 | 5.9% | 19.5% |
-| **3-gram** | Subword | 2,726 | 11.41 | 81,142 | 23.8% | 68.1% |
-| **4-gram** | Word | 279,047 | 18.09 | 973,376 | 4.1% | 14.8% |
-| **4-gram** | Subword | 17,313 | 14.08 | 421,873 | 10.8% | 35.5% |
+| **2-gram** | Word | 37,064 | 15.18 | 295,690 | 13.7% | 32.1% |
+| **2-gram** | Subword | 293 🏆 | 8.19 | 11,777 | 65.4% | 98.9% |
+| **3-gram** | Word | 127,942 | 16.97 | 571,162 | 5.9% | 19.5% |
+| **3-gram** | Subword | 2,712 | 11.41 | 80,865 | 23.9% | 68.2% |
+| **4-gram** | Word | 277,916 | 18.08 | 975,958 | 4.1% | 14.9% |
+| **4-gram** | Subword | 17,204 | 14.07 | 420,279 | 10.8% | 35.6% |
+| **5-gram** | Word | 202,294 | 17.63 | 684,204 | 4.9% | 16.7% |
+| **5-gram** | Subword | 72,650 | 16.15 | 1,308,264 | 6.0% | 21.7% |
 
 ### Top 5 N-grams by Size
 
@@ -150,68 +162,88 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `e voe` | 59,782 |
-| 2 | `ar c` | 55,139 |
-| 3 | `a viz` | 53,711 |
-| 4 | `e oa` | 52,022 |
-| 5 | `d ar` | 47,935 |
+| 1 | `e voe` | 60,584 |
+| 2 | `ar c` | 55,004 |
+| 3 | `a viz` | 53,947 |
+| 4 | `e oa` | 52,533 |
+| 5 | `d ar` | 48,158 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `zo ur gumun` | 17,678 |
-| 2 | `bro c hall` | 15,638 |
-| 3 | `a zo ur` | 15,315 |
-| 4 | `e oa bet` | 12,845 |
-| 5 | `ur gumun eus` | 8,897 |
+| 1 | `zo ur gumun` | 17,679 |
+| 2 | `bro c hall` | 15,683 |
+| 3 | `a zo ur` | 15,380 |
+| 4 | `e oa bet` | 13,023 |
+| 5 | `ur gumun eus` | 8,893 |
 
 **4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `zo ur gumun eus` | 8,261 |
-| 2 | `monumantoù ha traoù heverk` | 5,435 |
+| 1 | `zo ur gumun eus` | 8,258 |
+| 2 | `monumantoù ha traoù heverk` | 5,437 |
 | 3 | `a zo ur gumun` | 5,065 |
-| 4 | `zo ur gumun e` | 4,314 |
-| 5 | `monumant ar re varv` | 3,991 |
+| 4 | `zo ur gumun e` | 4,316 |
+| 5 | `monumant ar re varv` | 3,982 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `a zo ur gumun eus` | 3,616 |
+| 2 | `ioc world bird list diwar` | 2,760 |
+| 3 | `world bird list diwar benn` | 2,760 |
+| 4 | `roadennoù ioc world bird list` | 2,759 |
+| 5 | `zo ur gumun eus italia` | 2,622 |
 
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `_ a` | 1,901,092 |
-| 2 | `_ e` | 1,675,345 |
-| 3 | `a n` | 1,608,231 |
-| 4 | `e _` | 1,592,896 |
-| 5 | `r _` | 1,428,493 |
+| 1 | `_ a` | 1,908,238 |
+| 2 | `_ e` | 1,681,083 |
+| 3 | `a n` | 1,609,135 |
+| 4 | `e _` | 1,599,725 |
+| 5 | `r _` | 1,429,762 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `a r _` | 640,562 |
-| 2 | `_ e _` | 639,818 |
-| 3 | `e t _` | 623,800 |
-| 4 | `_ a r` | 555,503 |
-| 5 | `e n n` | 467,995 |
+| 1 | `a r _` | 641,927 |
+| 2 | `_ e _` | 641,853 |
+| 3 | `e t _` | 627,577 |
+| 4 | `_ a r` | 556,810 |
+| 5 | `e n n` | 468,710 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `_ a r _` | 456,425 |
-| 2 | `_ a n _` | 280,111 |
-| 3 | `a n t _` | 269,237 |
-| 4 | `_ g a n` | 228,714 |
-| 5 | `_ h a _` | 221,977 |
+| 1 | `_ a r _` | 457,578 |
+| 2 | `_ a n _` | 280,457 |
+| 3 | `a n t _` | 268,610 |
+| 4 | `_ g a n` | 228,380 |
+| 5 | `_ h a _` | 223,259 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `_ g a n t` | 202,257 |
+| 2 | `g a n t _` | 193,123 |
+| 3 | `_ h a g _` | 134,751 |
+| 4 | `_ e u s _` | 130,235 |
+| 5 | `e t _ e _` | 103,216 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram (subword) with 294
+- **Best Perplexity:** 2-gram (subword) with 293
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~35% of corpus
+- **Coverage:** Top-1000 patterns cover ~22% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.8900 | 1.853 | 7.60 | 546,189 | 11.0% |
-| **1** | Subword | 0.8953 | 1.860 | 5.85 | 8,402 | 10.5% |
-| **2** | Word | 0.3300 | 1.257 | 2.04 | 4,129,549 | 67.0% |
-| **2** | Subword | 0.6669 | 1.588 | 4.21 | 49,100 | 33.3% |
-| **3** | Word | 0.1561 | 1.114 | 1.34 | 8,377,190 | 84.4% |
-| **3** | Subword | 0.6656 | 1.586 | 3.74 | 206,555 | 33.4% |
-| **4** | Word | 0.0728 🏆 | 1.052 | 1.13 | 11,216,136 | 92.7% |
-| **4** | Subword | 0.6497 | 1.569 | 3.22 | 772,246 | 35.0% |
+| **1** | Word | 0.8873 | 1.850 | 7.57 | 546,965 | 11.3% |
+| **1** | Subword | 0.8951 | 1.860 | 5.84 | 8,419 | 10.5% |
+| **2** | Word | 0.3297 | 1.257 | 2.04 | 4,120,028 | 67.0% |
+| **2** | Subword | 0.6667 | 1.587 | 4.20 | 49,174 | 33.3% |
+| **3** | Word | 0.1564 | 1.115 | 1.35 | 8,357,037 | 84.4% |
+| **3** | Subword | 0.6634 | 1.584 | 3.73 | 206,424 | 33.7% |
+| **4** | Word | 0.0731 🏆 | 1.052 | 1.13 | 11,199,579 | 92.7% |
+| **4** | Subword | 0.6489 | 1.568 | 3.22 | 770,069 | 35.1% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `e breizh e vennozhioù diskouez an 18 muhelder bihanañ 34 evel caravan palace da vezañ ivez`
-2. `ar zastava e amzer tredeoged betek da vare a eskemm ha pterosaurus petra a ra an`
-3. `a oa un heñvelster gant dean bounce prison gang crime deep grand prix de france bleu`
+1. `e kastell aigneaux kantved merc h kannidi o devoa kemeret hent reter menezioù ezhomm da vont`
+2. `ar boblañs melestradurezh tud ar pif gadget de carnac et seigneur isaac baron met breinet gant`
+3. `a ra eus bro c haokaz ar fedon ar 25vet rujumant troadegiezhfichenn hiniennel memorial genweb egile`
 
 **Context Size 2:**
 
-1. `e voe embannet an testennoù klasel e vez da 800 000 den pe gant kraf ar revelezh`
-2. `ar c hembraeg mawr bras tolkien avat en doa bet ur gwadliñvadur e barzh ar c haramel`
-3. `a viz eost a oa ul livour hag un impalaer e voe kumun kernitron al lann e`
+1. `e voe azoet an oferenn rak miret eo bet troet e galleg a 346 pajennad a zeuas`
+2. `ar c haner en deus kumuniezhioù kumunioù beg ar skeul mañ zo levezonet gant friedrich dürrenmatt d`
+3. `a viz eost e departamant il ha gwilen bro roazhon bet ganet d ar mare se e`
 
 **Context Size 3:**
 
-1. `zo ur gumun en italia e proviñs cuneo etre kumunioù entracque ha valdieri anezhañ unan eus ugent lev...`
-2. `a zo ur bronneg geotdebrer a vev er meurvor atlantel belle isle distaget bɛl ˈaɪl e saozneg zo`
-3. `bro c hall zo un tiern ag ar morioù a zo e kenver ar sonerezh met evel ul`
+1. `zo ur gumun e spagn e kumuniezh valencia spagn pennadoù kar carlo ii charlez iañ karl iañ carlo`
+2. `a zo ur sammad a stennadur a en em astenn a ra erv kourland eus ledenez sambia lec`
+3. `bro c hall société des amis de benjamin péret pour un second manifeste communiste gant grandizo muni...`
 
 **Context Size 4:**
 
-1. `zo ur gumun eus meurgêr palermo e sikilia un 3 400 a dud zo enni o chom anezhi an`
-2. `monumantoù ha traoù heverk iliz katolik saint eustacheclochers de france douaroniezh emdroadur ar bo...`
-3. `a zo ur gumun eus italia e proviñs piacenza e rannvro emilia romagna ha 525 940 a dud o`
+1. `zo ur gumun eus departamant calvados e bro c hall douaroniezh armerzh emdroadur ar boblañs melestrad...`
+2. `monumantoù ha traoù heverk iliz katolik sant albin ners douaroniezh emdroadur ar boblañs cassini hag...`
+3. `a zo ur gumun eus departamant pas de calais bro c hall istor armerzh kompagnunezh mengleuzioù bruay ...`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `_t_onur_b_el_gan`
-2. `er_pakoumiat_g_v`
-3. `adoù-stiz_aleuri`
+1. `_cheunoù_wez:_be`
+2. `ere_zharndütren_`
+3. `añs_t.lalel_da_k`
 
 **Context Size 2:**
 
-1. `_a_un_erl_ezenner`
-2. `_e_he_c'hhn_/_niz`
-3. `an_ero_liged_;_ev`
+1. `_amm_da_gant_ges_`
+2. `_evez._marezal_pe`
+3. `annoù_art,_pag_ga`
 
 **Context Size 3:**
 
-1. `ar_bretek_ilioù,_o`
-2. `_e_pyrrarkva,_leon`
-3. `et_gant_franne,_ga`
+1. `ar_senner._levelet`
+2. `_e_rout_-_bloareku`
+3. `et_en_affarink_d’a`
 
 **Context Size 4:**
 
-1. `_ar_spesadoù_war_ar`
-2. `_an_emsavid_fy_nhad`
-3. `ant_an_alamanentiad`
+1. `_ar_solinago,_mab_s`
+2. `_an_ilizoù_sir_krei`
+3. `ant_bet_kemeret_an_`
 
 
 ### Key Findings
 
 - **Best Predictability:** Context-4 (word) with 92.7% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (772,246 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (770,069 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,64 +346,64 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 242,115 |
-| Total Tokens | 15,327,088 |
-| Mean Frequency | 63.30 |
+| Vocabulary Size | 241,991 |
+| Total Tokens | 15,343,130 |
+| Mean Frequency | 63.40 |
 | Median Frequency | 4 |
-| Frequency Std Dev | 2500.68 |
+| Frequency Std Dev | 2509.84 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | e | 701,948 |
-| 2 | ar | 517,584 |
-| 3 | a | 464,667 |
-| 4 | an | 326,300 |
-| 5 | ha | 228,454 |
-| 6 | gant | 189,759 |
-| 7 | c | 186,830 |
-| 8 | en | 181,309 |
-| 9 | da | 170,732 |
-| 10 | ur | 158,708 |
+| 1 | e | 703,890 |
+| 2 | ar | 518,682 |
+| 3 | a | 468,243 |
+| 4 | an | 326,691 |
+| 5 | ha | 229,662 |
+| 6 | gant | 189,178 |
+| 7 | c | 187,433 |
+| 8 | en | 180,997 |
+| 9 | da | 171,218 |
+| 10 | ur | 158,920 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | nfpb | 2 |
-| 2 | konjic | 2 |
-| 3 | formoraich | 2 |
-| 4 | vsn | 2 |
-| 5 | moldavie | 2 |
-| 6 | yankovich | 2 |
-| 7 | gueydon | 2 |
-| 8 | tréhouart | 2 |
-| 9 | bouguen | 2 |
-| 10 | shimosa | 2 |
+| 1 | veyne | 2 |
+| 2 | wga | 2 |
+| 3 | codreanu | 2 |
+| 4 | dumitru | 2 |
+| 5 | maghrebonkoud | 2 |
+| 6 | fidefide | 2 |
+| 7 | ougandachess | 2 |
+| 8 | cytonn | 2 |
+| 9 | malinga | 2 |
+| 10 | ablainville | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 1.1106 |
-| R² (Goodness of Fit) | 0.996763 |
+| Zipf Coefficient | 1.1114 |
+| R² (Goodness of Fit) | 0.996756 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 41.7% |
+| Top 100 | 41.9% |
 | Top 1,000 | 65.8% |
-| Top 5,000 | 80.4% |
-| Top 10,000 | 85.6% |
+| Top 5,000 | 80.5% |
+| Top 10,000 | 85.7% |
 
 ### Key Findings
 
 - **Zipf Compliance:** R²=0.9968 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 41.7% of corpus
-- **Long Tail:** 232,115 words needed for remaining 14.4% coverage
+- **High Frequency Dominance:** Top 100 words cover 41.9% of corpus
+- **Long Tail:** 231,991 words needed for remaining 14.3% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.8120 | 0.3605 | N/A | N/A |
-| **mono_64d** | 64 | 0.8171 🏆 | 0.2761 | N/A | N/A |
-| **mono_128d** | 128 | 0.7922 | 0.2119 | N/A | N/A |
+| **mono_32d** | 32 | 0.8117 | 0.3810 | N/A | N/A |
+| **mono_64d** | 64 | 0.8154 🏆 | 0.2792 | N/A | N/A |
+| **mono_128d** | 128 | 0.8010 | 0.2076 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8117 | 0.3700 | 0.2440 | 0.6460 |
+| **aligned_64d** | 64 | 0.8154 | 0.2752 | 0.3920 | 0.7600 |
+| **aligned_128d** | 128 | 0.8010 | 0.2094 | 0.5340 | 0.8640 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_64d with 0.8171 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.2828. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** mono_64d with 0.8154 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.2871. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 53.4% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **-0.232** | Low formulaic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -430,12 +465,11 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-s` | mariånas, gilgamès, battalions |
-| `-er` | tufer, hutier, beaver |
-| `-où` | damkanadoù, barrennoù, heitioù |
-| `-es` | tarbes, marcondes, cordes |
-| `-us` | luchinus, menenius, fulmarus |
-| `-en` | weyden, minchen, wageningen |
+| `-s` | wolves, hobbs, cassis |
+| `-où` | gwallzarvoudoù, emstummoù, pellgomzioù |
+| `-us` | tarphonomus, benildus, gigantorhinus |
+| `-er` | hompozer, siger, geschwister |
+| `-es` | wolves, béssèges, fontenailles |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -443,18 +477,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `tion` | 2.51x | 77 contexts | tione, metion, aktion |
-| `emba` | 2.14x | 41 contexts | pemba, emban, bemba |
-| `adoù` | 1.80x | 74 contexts | dadoù, kadoù, zadoù |
-| `nnet` | 1.78x | 70 contexts | annet, bonnet, linnet |
+| `tion` | 2.41x | 78 contexts | tione, eetion, motion |
+| `adoù` | 2.03x | 74 contexts | tadoù, padoù, hadoù |
+| `emba` | 2.26x | 40 contexts | emban, pemba, bemba |
 | `iamm` | 2.35x | 24 contexts | liamm, fiamma, fiamme |
-| `ouar` | 1.48x | 126 contexts | douar, zouar, mouar |
-| `nnad` | 1.52x | 97 contexts | bennad, rannad, hannad |
-| `zhio` | 1.87x | 40 contexts | uzhioù, lezhioù, bezhioù |
-| `zhañ` | 1.91x | 35 contexts | ezhañ, kozhañ, dizhañ |
-| `nnoù` | 1.84x | 39 contexts | tennoù, vannoù, bennoù |
-| `hone` | 1.81x | 40 contexts | honeg, khone, dhone |
-| `reze` | 1.46x | 94 contexts | breze, dreze, rezet |
+| `ouar` | 1.52x | 126 contexts | mouar, zouar, bouar |
+| `nnet` | 1.68x | 71 contexts | annet, rannet, rennet |
+| `nnad` | 1.53x | 98 contexts | mennad, gannad, vennad |
+| `zhañ` | 1.96x | 35 contexts | ezhañ, tizhañ, dizhañ |
+| `reze` | 1.52x | 94 contexts | rezet, dreze, breze |
+| `ntañ` | 1.75x | 51 contexts | antaño, vontañ, wintañ |
+| `nnoù` | 1.87x | 38 contexts | vannoù, gennoù, pennoù |
+| `iwar` | 2.55x | 13 contexts | diwar, ziwar, siward |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -469,26 +503,26 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
-| eildelwennoù | **`eildelwenn-où`** | 4.5 | `eildelwenn` |
-| hejadennoù | **`hejadenn-où`** | 4.5 | `hejadenn` |
-| wissenschaften | **`wissenschaft-en`** | 4.5 | `wissenschaft` |
-| beauvaisen | **`beauvais-en`** | 4.5 | `beauvais` |
-| wellaennoù | **`wellaenn-où`** | 4.5 | `wellaenn` |
-| antoninus | **`antonin-us`** | 4.5 | `antonin` |
-| pluñvennoù | **`pluñvenn-où`** | 4.5 | `pluñvenn` |
-| kementadoù | **`kementad-où`** | 4.5 | `kementad` |
-| compositores | **`compositor-es`** | 4.5 | `compositor` |
-| garidelloù | **`garidell-où`** | 4.5 | `garidell` |
-| hromozomoù | **`hromozom-où`** | 4.5 | `hromozom` |
-| reolennoù | **`reolenn-où`** | 4.5 | `reolenn` |
-| barringer | **`barring-er`** | 4.5 | `barring` |
-| diamantes | **`diamant-es`** | 4.5 | `diamant` |
-| stradivarius | **`stradivari-us`** | 4.5 | `stradivari` |
+| heureuses | **`heure-us-es`** | 6.0 | `heure` |
+| burzhudoù | **`burzhud-où`** | 4.5 | `burzhud` |
+| ziarbennoù | **`ziarbenn-où`** | 4.5 | `ziarbenn` |
+| goudeskridoù | **`goudeskrid-où`** | 4.5 | `goudeskrid` |
+| nijadegoù | **`nijadeg-où`** | 4.5 | `nijadeg` |
+| ziskoulmoù | **`ziskoulm-où`** | 4.5 | `ziskoulm` |
+| dasprenus | **`daspren-us`** | 4.5 | `daspren` |
+| tradutores | **`tradutor-es`** | 4.5 | `tradutor` |
+| drubuilhoù | **`drubuilh-où`** | 4.5 | `drubuilh` |
+| reichsmarkoù | **`reichsmark-où`** | 4.5 | `reichsmark` |
+| variantennoù | **`variantenn-où`** | 4.5 | `variantenn` |
+| livuzennoù | **`livuzenn-où`** | 4.5 | `livuzenn` |
+| kompozadoù | **`kompozad-où`** | 4.5 | `kompozad` |
+| viñsaskelloù | **`viñsaskell-où`** | 4.5 | `viñsaskell` |
+| kellennoù | **`kellenn-où`** | 4.5 | `kellenn` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language BR appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Breton shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
 
 ---
 ## 7. Summary & Recommendations
@@ -500,7 +534,7 @@ The language BR appears to be more isolating or has a highly fixed vocabulary. W
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
 | Tokenizer | **64k BPE** | Best compression (3.79x) |
-| N-gram | **2-gram** | Lowest perplexity (294) |
+| N-gram | **2-gram** | Lowest perplexity (293) |
 | Markov | **Context-4** | Highest predictability (92.7%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
@@ -715,4 +749,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 08:48:16*
+*Report Date: 2026-01-03 20:37:28*

models/embeddings/aligned/br_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cf7029aeb871b95976d657332c7979955b3472f5cc997911ea5845326eed098
+size 1181336739

models/embeddings/aligned/br_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "br", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/br_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8019b49dd16fa1eaea2d6883a25739f54360b9485c5721e59f5b853a7e93eca8
+size 65664

models/embeddings/aligned/br_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "br",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 63876,
+  "vocab_size": 151042
+}

models/embeddings/aligned/br_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5a04434da293f96ee1cf3159f4953ae9e70374e388480d3f97da4466cad588f
+size 297336483

models/embeddings/aligned/br_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "br", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/br_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c830997beb7d49617d22238749dab4c67739138aadce1e34c8d4ab91871baafd
+size 4224

models/embeddings/aligned/br_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "br",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 63876,
+  "vocab_size": 151042
+}

models/embeddings/aligned/br_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38a0702c9382196b1aa4715b46199c5a1a04dc269571320f1da9c5b35eed7285
+size 592003235

models/embeddings/aligned/br_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "br", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/br_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e234edf9a9b2ba2bdd409b023eaafcb3f40bcaf318174e7fc979aa6fa32277b5
+size 16512

models/embeddings/aligned/br_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "br",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 63876,
+  "vocab_size": 151042
+}

models/embeddings/monolingual/br_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2cc2dcf0e5dbf7b0a47f5b7b84203934a4ef4a94c793c7cf148e47bcf6831e94
-size 1181713085
+oid sha256:9cf7029aeb871b95976d657332c7979955b3472f5cc997911ea5845326eed098
+size 1181336739

models/embeddings/monolingual/br_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 151402
+  "vocab_size": 151042
 }

models/embeddings/monolingual/br_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:563c972d1b01c4444517fc5d7b7951b8d9f5cf9c450bf8bfa161f4a7504bc953
-size 297436349
+oid sha256:a5a04434da293f96ee1cf3159f4953ae9e70374e388480d3f97da4466cad588f
+size 297336483

models/embeddings/monolingual/br_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 151402
+  "vocab_size": 151042
 }

models/embeddings/monolingual/br_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:64947ec6dfc340b15fe1245dfbf27eb885378a9c32be40f616fce71e20a67416
-size 592195261
+oid sha256:38a0702c9382196b1aa4715b46199c5a1a04dc269571320f1da9c5b35eed7285
+size 592003235

models/embeddings/monolingual/br_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 151402
+  "vocab_size": 151042
 }

models/subword_markov/br_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:364c87eeed84e49a322b272f9060cbaf46e7cf91d5fc9cf833ebcd76d814171e
-size 371383
+oid sha256:76e6fae95f5a61fb627737e66053e388ea5468f419a8ba3f08957a4a4383bec5
+size 372500

models/subword_markov/br_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "br",
-  "unique_contexts": 8402,
-  "total_transitions": 88562222
+  "unique_contexts": 8419,
+  "total_transitions": 88614190
 }

models/subword_markov/br_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b2ffa4ddd6ad6edd05335dca480707c3b64c0a14c509081d1d74dfe72edc981
-size 1665166
+oid sha256:3dd84b5d12129dd75fbbc2ec5327bc62e709e34ec5ff585646adf2844a3251ab
+size 1701096

models/subword_markov/br_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "br",
-  "unique_contexts": 49100,
-  "total_transitions": 88473399
+  "unique_contexts": 49174,
+  "total_transitions": 88524855
 }

models/subword_markov/br_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:692d3af21734dce1c8cf7fcea827b781d47ef5c8b25654782ff6145d753309c2
-size 6642409
+oid sha256:ccc7aec735112a7272c9a1b866452f6de1fc15a32bc592b84c7c2954d8565f0d
+size 6611427

models/subword_markov/br_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "br",
-  "unique_contexts": 206555,
-  "total_transitions": 88384576
+  "unique_contexts": 206424,
+  "total_transitions": 88435520
 }

models/subword_markov/br_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:02fe0d42cac9d5c8869d073c8570acc4ae0f693f23327c55de4b987d7ddb8e52
-size 20533909
+oid sha256:9d1c38f7192e08d688c3c4b1b4d47f77efc324b960e025aaf54c352f7c68a3af
+size 20565163

models/subword_markov/br_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "br",
-  "unique_contexts": 772246,
-  "total_transitions": 88295753
+  "unique_contexts": 770069,
+  "total_transitions": 88346185
 }

models/subword_ngram/br_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7aa9a0309201a5bff85bb732039ec09cee34921527090479840454db58c6414f
-size 154793
+oid sha256:6100bec355398114589c1a591ef7cce0aa74534699fa7dcd3bc536e8434f08fd
+size 154502

models/subword_ngram/br_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "br",
-  "unique_ngrams": 11776,
-  "total_ngrams": 88562222
+  "unique_ngrams": 11777,
+  "total_ngrams": 88614190
 }

models/subword_ngram/br_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f11838b2e1dd3db7e5060c435f0ed9b2d633720992bd49d8414dc3ebb1947f1
-size 1025779
+oid sha256:23ba302a7d19117f6743e8131a78e057e141e0319cb42d9352ce8baf3d888b02
+size 1028682

models/subword_ngram/br_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "br",
-  "unique_ngrams": 81142,
-  "total_ngrams": 88473399
+  "unique_ngrams": 80865,
+  "total_ngrams": 88524855
 }

models/subword_ngram/br_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0662a30ed0f866ba0ef46fba70681b5e219c20b4d5f818869aeaf346ad95cea
-size 4921369
+oid sha256:ed9eaf1dc00b86ec9e962f9ce0fab70ded47d94549dfe615cc34515caa8d2149
+size 4904958

models/subword_ngram/br_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "br",
-  "unique_ngrams": 421873,
-  "total_ngrams": 88384576
+  "unique_ngrams": 420279,
+  "total_ngrams": 88435520
 }

models/subword_ngram/br_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d24c8a32466633aad19b5685f4f96e93beed6daf68754b6913a335d6f3be528
+size 15681978

models/subword_ngram/br_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "br",
+  "unique_ngrams": 1308264,
+  "total_ngrams": 88346185
+}

models/tokenizer/br_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a34b2820290797923a2b18aa49d5154f1b7c7d12b7564cace04d896d97c6e53
-size 502646
+oid sha256:83fb88fe9bf797ec7100535d626d2281591253ac7a7192ec8f714ef1275449bc
+size 502780

models/tokenizer/br_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:26ea0db3a85a9e9055b43984cf4ac687e795f5825553ccc302a5abc29d9b8273
-size 774769
+oid sha256:911df452c5729a9800392f02d004ab34704c39363199a59b81c398dc85d0ae9d
+size 775074

models/tokenizer/br_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d3d3d03f79fa86777af32c2e613a308e2b7c627fbf49f5d7a3c97678ebafad13
-size 1335604
+oid sha256:1c4bb37ad9f7da618a62705a639a7e995aa500aac4d54128b97c2e200a0c9286
+size 1336555

models/tokenizer/br_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6a1ce4f9267e8a3f8951e0d2dfd0751006c25871dfdcd890cceebf4f01248eb
-size 370491
+oid sha256:416bd093915a7ce9b3d37f55d4e6f19b655bc9cb93ca1ad42e442fda1839cb70
+size 370573

models/vocabulary/br_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ace7ffc403a7ed53e5646074aa9d3b0acceee4c23eb067355b372c233b0488f
-size 3765893
+oid sha256:9853208ecd8f588061cdc38e2b3046ab1d262a5ec3dfd25bf4a60397033b1b26
+size 3762757

models/vocabulary/br_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "br",
-  "vocabulary_size": 242115,
+  "vocabulary_size": 241991,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.03497837029849385,
+    "type_token_ratio": 0.034990028236873805,
     "coverage": {
-      "top_100": 0.40905545676087623,
-      "top_1000": 0.6447280425360034,
-      "top_5000": 0.7885968080597011,
-      "top_10000": 0.8392496909814169
+      "top_100": 0.41033685445941886,
+      "top_1000": 0.6455537132284644,
+      "top_5000": 0.7892532453361742,
+      "top_10000": 0.8397786344630703
     },
-    "hapax_count": 304658,
-    "hapax_ratio": 0.5571928387100314,
-    "total_documents": 88823
+    "hapax_count": 305557,
+    "hapax_ratio": 0.5580460525835178,
+    "total_documents": 89335
   }
 }

models/word_markov/br_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be693644d5a0ebbb01f9e30236eea7c454a9d88268b56513e574925b7ff00051
-size 35952141
+oid sha256:de721e8dee95a4e4ce94021f78ae05a169c11550ad6b494d9e6627b8b2c68df2
+size 35934580

models/word_markov/br_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "br",
-  "unique_contexts": 546189,
-  "total_transitions": 15542923
+  "unique_contexts": 546965,
+  "total_transitions": 15559352
 }

models/word_markov/br_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8fda1c756cb7fc2b4ab6e70c2cd1d67819d416d7e280f2e0548839fb1da8d2f
-size 97354526
+oid sha256:25eda57d96503b9d2a3e57951e6143e326c6fd10fdb26ddde28002172acb1f49
+size 97522977

models/word_markov/br_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "br",
-  "unique_contexts": 4129549,
-  "total_transitions": 15454100
+  "unique_contexts": 4120028,
+  "total_transitions": 15470017
 }