Upload all models and assets for bcl (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: bcl
-language_name: BCL
+language_name: Central Bikol
 language_family: austronesian_philippine_central
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-austronesian_philippine_central
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.812
+    value: 4.810
   - name: best_isotropy
     type: isotropy
-    value: 0.8253
+    value: 0.8247
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# BCL - Wikilangs Models
+# Central Bikol - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **BCL** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Central Bikol** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.956x | 3.96 | 0.0173% | 358,080 |
-| **16k** | 4.291x | 4.29 | 0.0188% | 330,176 |
-| **32k** | 4.574x | 4.58 | 0.0200% | 309,738 |
-| **64k** | 4.812x 🏆 | 4.82 | 0.0211% | 294,409 |
+| **8k** | 3.957x | 3.96 | 0.0152% | 354,491 |
+| **16k** | 4.291x | 4.29 | 0.0165% | 326,860 |
+| **32k** | 4.572x | 4.58 | 0.0176% | 306,791 |
+| **64k** | 4.810x 🏆 | 4.81 | 0.0185% | 291,605 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Si Magno "Carlo" Jose Caparas (Marso 12, sa Pampanga - Mayo 25, sarong paragibon...`
+**Sample 1:** `An sarong taon sa Gregoryanong kalendaryo. Enero Pebrero Marso Abril Mayo Hunyo ...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁si ▁mag no ▁" car lo " ▁jose ▁cap aras ... (+31 more)` | 41 |
-| 16k | `▁si ▁mag no ▁" car lo " ▁jose ▁cap aras ... (+28 more)` | 38 |
-| 32k | `▁si ▁magno ▁" carlo " ▁jose ▁caparas ▁( marso ▁ ... (+25 more)` | 35 |
-| 64k | `▁si ▁magno ▁" carlo " ▁jose ▁caparas ▁( marso ▁ ... (+25 more)` | 35 |
+| 8k | `▁an ▁sarong ▁taon ▁sa ▁gregoryanong ▁kalendaryo . ▁enero ▁pebrero ▁marso ... (+9 more)` | 19 |
+| 16k | `▁an ▁sarong ▁taon ▁sa ▁gregoryanong ▁kalendaryo . ▁enero ▁pebrero ▁marso ... (+9 more)` | 19 |
+| 32k | `▁an ▁sarong ▁taon ▁sa ▁gregoryanong ▁kalendaryo . ▁enero ▁pebrero ▁marso ... (+9 more)` | 19 |
+| 64k | `▁an ▁sarong ▁taon ▁sa ▁gregoryanong ▁kalendaryo . ▁enero ▁pebrero ▁marso ... (+9 more)` | 19 |
 
-**Sample 2:** `An Vermont sarong estado kan Estados Unidos. Kataytayan nin mga ladawan estado k...`
+**Sample 2:** `Si Donald James "Donny" Lucas (Montreal) dating sarong Amerikanong entertainer.`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁an ▁ver m ont ▁sarong ▁estado ▁kan ▁estados ▁unidos . ... (+8 more)` | 18 |
-| 16k | `▁an ▁ver mont ▁sarong ▁estado ▁kan ▁estados ▁unidos . ▁kataytayan ... (+7 more)` | 17 |
-| 32k | `▁an ▁vermont ▁sarong ▁estado ▁kan ▁estados ▁unidos . ▁kataytayan ▁nin ... (+6 more)` | 16 |
-| 64k | `▁an ▁vermont ▁sarong ▁estado ▁kan ▁estados ▁unidos . ▁kataytayan ▁nin ... (+6 more)` | 16 |
+| 8k | `▁si ▁d onald ▁james ▁" don ny " ▁luc as ... (+10 more)` | 20 |
+| 16k | `▁si ▁donald ▁james ▁" don ny " ▁lucas ▁( mont ... (+7 more)` | 17 |
+| 32k | `▁si ▁donald ▁james ▁" don ny " ▁lucas ▁( mont ... (+7 more)` | 17 |
+| 64k | `▁si ▁donald ▁james ▁" don ny " ▁lucas ▁( mont ... (+7 more)` | 17 |
 
-**Sample 3:** `An sarong komyun asin banwaan sa Provincia nin Frosinone sa rehiyon Lazio kan It...`
+**Sample 3:** `An Yenon sarong baryo sa Abi na lugar kan gobyerno lokal sa Cross River State, N...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁an ▁sarong ▁komyun ▁asin ▁banwaan ▁sa ▁provincia ▁nin ▁f rosin ... (+7 more)` | 17 |
-| 16k | `▁an ▁sarong ▁komyun ▁asin ▁banwaan ▁sa ▁provincia ▁nin ▁frosinone ▁sa ... (+5 more)` | 15 |
-| 32k | `▁an ▁sarong ▁komyun ▁asin ▁banwaan ▁sa ▁provincia ▁nin ▁frosinone ▁sa ... (+5 more)` | 15 |
-| 64k | `▁an ▁sarong ▁komyun ▁asin ▁banwaan ▁sa ▁provincia ▁nin ▁frosinone ▁sa ... (+5 more)` | 15 |
+| 8k | `▁an ▁y en on ▁sarong ▁baryo ▁sa ▁ab i ▁na ... (+18 more)` | 28 |
+| 16k | `▁an ▁y en on ▁sarong ▁baryo ▁sa ▁ab i ▁na ... (+17 more)` | 27 |
+| 32k | `▁an ▁yen on ▁sarong ▁baryo ▁sa ▁abi ▁na ▁lugar ▁kan ... (+15 more)` | 25 |
+| 64k | `▁an ▁yen on ▁sarong ▁baryo ▁sa ▁abi ▁na ▁lugar ▁kan ... (+15 more)` | 25 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.812x compression
-- **Lowest UNK Rate:** 8k with 0.0173% unknown tokens
+- **Best Compression:** 64k achieves 4.810x compression
+- **Lowest UNK Rate:** 8k with 0.0152% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 29,761 | 14.86 | 138,758 | 13.5% | 31.1% |
-| **2-gram** | Subword | 216 🏆 | 7.75 | 6,792 | 72.6% | 99.3% |
-| **3-gram** | Word | 80,221 | 16.29 | 216,640 | 7.6% | 19.4% |
-| **3-gram** | Subword | 1,808 | 10.82 | 46,201 | 33.1% | 73.8% |
-| **4-gram** | Word | 126,144 | 16.94 | 300,994 | 9.3% | 17.1% |
-| **4-gram** | Subword | 10,403 | 13.34 | 248,296 | 18.8% | 43.7% |
+| **2-gram** | Word | 29,762 | 14.86 | 139,543 | 13.5% | 31.1% |
+| **2-gram** | Subword | 215 🏆 | 7.75 | 6,829 | 72.7% | 99.3% |
+| **3-gram** | Word | 81,081 | 16.31 | 219,146 | 7.5% | 19.3% |
+| **3-gram** | Subword | 1,801 | 10.81 | 46,307 | 33.2% | 73.8% |
+| **4-gram** | Word | 128,131 | 16.97 | 304,782 | 9.2% | 17.0% |
+| **4-gram** | Subword | 10,353 | 13.34 | 249,114 | 18.9% | 43.8% |
+| **5-gram** | Word | 55,135 | 15.75 | 164,721 | 16.0% | 24.8% |
+| **5-gram** | Subword | 39,111 | 15.26 | 711,663 | 11.0% | 29.6% |
 
 ### Top 5 N-grams by Size
 
@@ -150,68 +162,88 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `sa mga` | 29,819 |
-| 2 | `an mga` | 26,719 |
-| 3 | `kan mga` | 22,256 |
-| 4 | `iyo an` | 17,168 |
-| 5 | `nin mga` | 16,442 |
+| 1 | `sa mga` | 30,516 |
+| 2 | `an mga` | 27,434 |
+| 3 | `kan mga` | 22,662 |
+| 4 | `iyo an` | 17,275 |
+| 5 | `nin mga` | 16,825 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `panluwas na takod` | 5,464 |
-| 2 | `mga panluwas na` | 4,866 |
-| 3 | `toltolan mga panluwas` | 2,765 |
-| 4 | `para sa mga` | 2,679 |
+| 1 | `panluwas na takod` | 5,506 |
+| 2 | `mga panluwas na` | 4,909 |
+| 3 | `toltolan mga panluwas` | 2,791 |
+| 4 | `para sa mga` | 2,778 |
 | 5 | `igwa ining sukol` | 2,227 |
 
 **4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `mga panluwas na takod` | 4,571 |
-| 2 | `toltolan mga panluwas na` | 2,765 |
+| 1 | `mga panluwas na takod` | 4,613 |
+| 2 | `toltolan mga panluwas na` | 2,791 |
 | 3 | `igwa ining sukol na` | 2,139 |
-| 4 | `philippine standard geographic code` | 1,750 |
+| 4 | `philippine standard geographic code` | 1,751 |
 | 5 | `sa sensus kan igwa` | 1,728 |
 
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `toltolan mga panluwas na takod` | 2,656 |
+| 2 | `sa sensus kan igwa ining` | 1,724 |
+| 3 | `standard geographic code local governance` | 1,722 |
+| 4 | `com philippine standard geographic code` | 1,722 |
+| 5 | `philatlas com philippine standard geographic` | 1,722 |
+
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `a n` | 1,344,298 |
-| 2 | `a _` | 1,288,104 |
-| 3 | `n _` | 1,218,286 |
-| 4 | `_ s` | 827,447 |
-| 5 | `n a` | 787,793 |
+| 1 | `a n` | 1,358,991 |
+| 2 | `a _` | 1,303,105 |
+| 3 | `n _` | 1,232,546 |
+| 4 | `_ s` | 834,968 |
+| 5 | `n a` | 797,325 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `a n _` | 694,503 |
-| 2 | `_ n a` | 534,019 |
-| 3 | `_ s a` | 519,575 |
-| 4 | `n g _` | 461,251 |
-| 5 | `_ k a` | 374,182 |
+| 1 | `a n _` | 702,654 |
+| 2 | `_ n a` | 541,439 |
+| 3 | `_ s a` | 524,860 |
+| 4 | `n g _` | 465,207 |
+| 5 | `_ k a` | 378,564 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `_ s a _` | 333,664 |
-| 2 | `_ n a _` | 329,330 |
-| 3 | `k a n _` | 234,296 |
-| 4 | `_ k a n` | 230,493 |
-| 5 | `_ a n _` | 210,822 |
+| 1 | `_ s a _` | 337,217 |
+| 2 | `_ n a _` | 333,981 |
+| 3 | `k a n _` | 236,687 |
+| 4 | `_ k a n` | 232,949 |
+| 5 | `_ a n _` | 213,433 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `_ k a n _` | 225,191 |
+| 2 | `_ m g a _` | 166,824 |
+| 3 | `_ n i n _` | 131,940 |
+| 4 | `a s i n _` | 125,892 |
+| 5 | `_ a s i n` | 125,534 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram (subword) with 216
+- **Best Perplexity:** 2-gram (subword) with 215
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~44% of corpus
+- **Coverage:** Top-1000 patterns cover ~30% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.7785 | 1.715 | 6.29 | 327,423 | 22.1% |
-| **1** | Subword | 0.9154 | 1.886 | 5.39 | 7,079 | 8.5% |
-| **2** | Word | 0.3185 | 1.247 | 1.98 | 2,054,215 | 68.1% |
-| **2** | Subword | 0.5355 | 1.449 | 3.36 | 38,137 | 46.5% |
-| **3** | Word | 0.1347 | 1.098 | 1.28 | 4,060,609 | 86.5% |
-| **3** | Subword | 0.6397 | 1.558 | 3.61 | 128,219 | 36.0% |
-| **4** | Word | 0.0494 🏆 | 1.035 | 1.08 | 5,171,638 | 95.1% |
-| **4** | Subword | 0.6483 | 1.567 | 3.06 | 463,318 | 35.2% |
+| **1** | Word | 0.7779 | 1.715 | 6.29 | 329,127 | 22.2% |
+| **1** | Subword | 0.9163 | 1.887 | 5.39 | 7,145 | 8.4% |
+| **2** | Word | 0.3186 | 1.247 | 1.99 | 2,064,138 | 68.1% |
+| **2** | Subword | 0.5336 | 1.448 | 3.35 | 38,469 | 46.6% |
+| **3** | Word | 0.1355 | 1.098 | 1.28 | 4,087,355 | 86.5% |
+| **3** | Subword | 0.6380 | 1.556 | 3.61 | 128,967 | 36.2% |
+| **4** | Word | 0.0498 🏆 | 1.035 | 1.08 | 5,215,534 | 95.0% |
+| **4** | Subword | 0.6487 | 1.568 | 3.06 | 465,409 | 35.1% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `sa sarong dating parakabayo na mitolohiya kan prepekturang hiroshima asin naglalaman nin estasyon pa...`
-2. `na binubuo an kapwa niya iyo an sityo sa filipinas pwesto kan mga panluwas na desenyo`
-3. `an elementong kimikal kaugalian na iran nag oogid nanggad nag aako sa vocals keyboards synths play`
+1. `sa tipan an apod na dinadalihigan kan taon kan komputasyon asin ipagbabalik sa tahaw kan taon`
+2. `na nag eeksister an mga mimetikong kalibangbang patag dakol na coronet an pahayag tanganing ipabisto...`
+3. `an mga komposisyon kan kompositor asin ngapit iyong watawat ang halaman asin gurutom suya sumo mga`
 
 **Context Size 2:**
 
-1. `sa mga libreriya sa unibersidad kan klima permanenteng binabago an inskripsiyon na gapo iyo nahahama...`
-2. `an mga heswita na si bruce lee tanganing magtukdo sa saiyang komunidad sa online campaign kan gabnet`
-3. `kan mga cyclopes mayo nin neutron an kasarosarong istruktura sa salog patapsco durante kan panahon n...`
+1. `sa mga minasunod the crucifixion saint anthony wisconsin si gross sarong multi partidong estado kata...`
+2. `an mga osipon sarong babaeng kustomer ining lalaki winaki siya nin labing 300 bilyon historya si jam...`
+3. `kan mga aldaw bago ini ibugtak sa sitwasyon kan halawig na kasaysayan asin sarong best seller asin`
 
 **Context Size 3:**
 
-1. `panluwas na takod philatlas com philippine standard geographic code local governance performance man...`
-2. `mga panluwas na takod the incorporated owners of chungking mansions sha tsui`
-3. `toltolan mga panluwas na gubing na ini parateng ibinubuntog sa sipon ini tanganing masigurado na pag...`
+1. `panluwas na takod opisyal na websityo toltolan paadalan sa kabikolan`
+2. `mga panluwas na takod philatlas com philippine standard geographic code local governance performance...`
+3. `toltolan mga panluwas na takod philatlas com philippine standard geographic code local governance pe...`
 
 **Context Size 4:**
 
-1. `mga panluwas na takod philatlas com philippine standard geographic code philippine census informatio...`
-2. `toltolan mga panluwas na takod philatlas com philippine standard geographic code local governance pe...`
-3. `igwa ining sukol na kilometro kwadrado an designadong zip code kaini iyo sosog sa sensus kan igwa in...`
+1. `mga panluwas na takod agi agi kan kawat na scrabblre kinua 06 11 16 mga bagay bagay dapit sa`
+2. `toltolan mga panluwas na takod si iu sa universal music japan koreanong artista`
+3. `igwa ining sukol na 173 70 kilometro kwadrado na kadagaan asin namumugtak sa ikaduwang distrito an d...`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `_ukikingama_ngam`
-2. `agrnan_ninin_n_i`
-3. `ntin_ag_teran_sw`
+1. `_c_naco'a_nimgho`
+2. `asinarosy_sig-em`
+3. `n,_kursud_wanari`
 
 **Context Size 2:**
 
-1. `angurehirin_mgank`
-2. `a_tawantenedyan._`
-3. `n_sin_of_ippelinc`
+1. `anta_pincion_they`
+2. `a_cagkan_kabong_i`
+3. `n_an_kahabaharopi`
 
 **Context Size 3:**
 
-1. `an_anahi_mode_nin_`
-2. `_na_le_pula_04:35_`
-3. `_sanriquerto_paan_`
+1. `an_sa_laog,_asin_l`
+2. `_na_at_sa_unra_san`
+3. `_sa_na_lugang_nin_`
 
 **Context Size 4:**
 
-1. `_sa_kaze_anggaro_sa`
-2. `_na_siness_(princia`
-3. `kan_cabulanguro_nin`
+1. `_sa_kastian_communi`
+2. `_na_dormasya_sa_pag`
+3. `kan_iban.[3]_an_sa_`
 
 
 ### Key Findings
 
-- **Best Predictability:** Context-4 (word) with 95.1% predictability
+- **Best Predictability:** Context-4 (word) with 95.0% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (463,318 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (465,409 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,64 +346,64 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 131,763 |
-| Total Tokens | 5,884,976 |
-| Mean Frequency | 44.66 |
+| Vocabulary Size | 132,282 |
+| Total Tokens | 5,940,352 |
+| Mean Frequency | 44.91 |
 | Median Frequency | 4 |
-| Frequency Std Dev | 1759.83 |
+| Frequency Std Dev | 1779.06 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | sa | 336,085 |
-| 2 | na | 332,599 |
-| 3 | an | 226,864 |
-| 4 | kan | 223,487 |
-| 5 | mga | 165,146 |
-| 6 | nin | 129,650 |
-| 7 | asin | 123,857 |
-| 8 | sarong | 61,956 |
-| 9 | si | 54,132 |
-| 10 | the | 42,788 |
+| 1 | sa | 339,632 |
+| 2 | na | 337,250 |
+| 3 | an | 230,137 |
+| 4 | kan | 225,822 |
+| 5 | mga | 168,493 |
+| 6 | nin | 132,058 |
+| 7 | asin | 125,726 |
+| 8 | sarong | 62,546 |
+| 9 | si | 54,313 |
+| 10 | the | 42,923 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | gorō | 2 |
-| 2 | amaji | 2 |
-| 3 | kasshi | 2 |
-| 4 | shukufuku | 2 |
-| 5 | teana | 2 |
-| 6 | siony | 2 |
-| 7 | keann | 2 |
-| 8 | libertadores | 2 |
-| 9 | rta | 2 |
-| 10 | kontoor | 2 |
+| 1 | akkuly | 2 |
+| 2 | sucuk | 2 |
+| 3 | zhaparova | 2 |
+| 4 | altynbekov | 2 |
+| 5 | wanatabe | 2 |
+| 6 | kordon | 2 |
+| 7 | sobringaran | 2 |
+| 8 | khanid | 2 |
+| 9 | ganish | 2 |
+| 10 | niceno | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 1.0202 |
-| R² (Goodness of Fit) | 0.994749 |
+| Zipf Coefficient | 1.0205 |
+| R² (Goodness of Fit) | 0.994695 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 43.2% |
-| Top 1,000 | 63.6% |
-| Top 5,000 | 79.3% |
+| Top 100 | 43.3% |
+| Top 1,000 | 63.7% |
+| Top 5,000 | 79.4% |
 | Top 10,000 | 85.4% |
 
 ### Key Findings
 
 - **Zipf Compliance:** R²=0.9947 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 43.2% of corpus
-- **Long Tail:** 121,763 words needed for remaining 14.6% coverage
+- **High Frequency Dominance:** Top 100 words cover 43.3% of corpus
+- **Long Tail:** 122,282 words needed for remaining 14.6% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.8253 🏆 | 0.3513 | N/A | N/A |
-| **mono_64d** | 64 | 0.8232 | 0.2638 | N/A | N/A |
-| **mono_128d** | 128 | 0.8182 | 0.1917 | N/A | N/A |
+| **mono_32d** | 32 | 0.8247 | 0.3483 | N/A | N/A |
+| **mono_64d** | 64 | 0.8238 | 0.2714 | N/A | N/A |
+| **mono_128d** | 128 | 0.8094 | 0.1968 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8247 🏆 | 0.3494 | 0.2280 | 0.5780 |
+| **aligned_64d** | 64 | 0.8238 | 0.2693 | 0.3700 | 0.7100 |
+| **aligned_128d** | 128 | 0.8094 | 0.1977 | 0.4780 | 0.8080 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.8253 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.2689. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** aligned_32d with 0.8247 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.2722. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 47.8% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **-0.162** | Low formulaic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -426,23 +461,24 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Prefixes
 | Prefix | Examples |
 |--------|----------|
-| `-pa` | parliamentarians, panribay, pagpaharong |
-| `-na` | nasipit, nagdesisyong, nakakalibog |
-| `-ma` | magsolnop, maiko, magdebut |
-| `-pag` | pagpaharong, pagkotkot, pagkasambit |
-| `-ka` | karella, kantada, kaneko |
-| `-nag` | nagdesisyong, nagkakampanyang, nagwawagayway |
-| `-pi` | pilian, pinaatras, pinagmaigotan |
+| `-pa` | pagraranggo, pandapog, pananakop |
+| `-na` | naquit, nagana, nagmamato |
+| `-ma` | maghelang, malos, mangyans |
+| `-pag` | pagraranggo, pagrehistro, pagsasalin |
+| `-pi` | pigrorokyaw, pigsaladawan, pigpapainitan |
+| `-nag` | nagana, nagmamato, nagashino |
+| `-ka` | kajaman, kalipunan, kambodya |
 
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-n` | rubinstein, hizen, ballon |
-| `-an` | sutan, tagiliran, pilian |
-| `-ng` | chaeryeong, issuing, sinkretikong |
-| `-on` | ballon, indemnipikasyon, monsoon |
-| `-ong` | chaeryeong, sinkretikong, pagpaharong |
-| `-ing` | issuing, isporting, nakakaheling |
+| `-n` | pigsaladawan, pigpapainitan, esperidion |
+| `-a` | smegma, emanuela, estrela |
+| `-ng` | maghelang, gyalwang, gansing |
+| `-an` | pigsaladawan, pigpapainitan, kajaman |
+| `-on` | esperidion, pasteurization, oryentasyon |
+| `-ong` | silensyong, mapabulong, otong |
+| `-ang` | maghelang, gyalwang, tatabang |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -450,18 +486,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `hili` | 2.57x | 38 contexts | chili, hilig, hilir |
-| `inak` | 2.14x | 68 contexts | pinak, inako, inakò |
-| `nter` | 1.96x | 91 contexts | inter, enter, antero |
-| `agka` | 1.87x | 107 contexts | pagka, magka, nagka |
-| `ista` | 1.82x | 115 contexts | pista, bista, lista |
-| `agpa` | 1.93x | 87 contexts | ragpa, agpay, pagpa |
-| `atio` | 2.05x | 51 contexts | patio, ratio, matios |
-| `nagp` | 2.38x | 25 contexts | nagpe, nagpa, nagpur |
-| `syon` | 1.80x | 72 contexts | bisyon, nasyon, posyon |
-| `kula` | 2.01x | 37 contexts | kulam, kulas, kulan |
-| `asyo` | 1.79x | 56 contexts | basyo, nasyo, hasyo |
-| `agin` | 1.89x | 44 contexts | sagin, aging, nagin |
+| `agka` | 1.94x | 108 contexts | pagka, nagka, magka |
+| `inak` | 2.14x | 67 contexts | inakô, inaka, inakò |
+| `atio` | 2.24x | 51 contexts | ratio, patio, matios |
+| `syon` | 2.04x | 72 contexts | mosyon, nasyon, losyon |
+| `agpa` | 1.87x | 88 contexts | ragpa, agpay, magpa |
+| `hili` | 2.23x | 39 contexts | hilig, chili, hilir |
+| `asyo` | 2.00x | 57 contexts | basyo, rasyo, nasyo |
+| `ista` | 1.67x | 114 contexts | istar, bista, istat |
+| `ndan` | 1.73x | 78 contexts | indan, ndang, andan |
+| `agin` | 1.84x | 44 contexts | sagin, magin, nagin |
+| `nagp` | 2.05x | 26 contexts | nagpe, nagpa, nagpur |
+| `embr` | 2.14x | 22 contexts | membro, embryo, myembro |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -469,16 +505,16 @@ This table shows which prefixes and suffixes most frequently co-occur on the sam
 
 | Prefix | Suffix | Frequency | Examples |
 |--------|--------|-----------|----------|
-| `-pa` | `-n` | 98 words | pagreparohon, patalingkason |
-| `-na` | `-n` | 86 words | nakaptan, naman |
-| `-ka` | `-n` | 81 words | kaaayon, kaenterohan |
-| `-na` | `-an` | 75 words | nakaptan, naman |
-| `-ka` | `-an` | 74 words | kaenterohan, kasilyasan |
-| `-pi` | `-n` | 70 words | pian, pinaomayan |
-| `-pi` | `-an` | 63 words | pian, pinaomayan |
-| `-pa` | `-an` | 59 words | patotoohan, panlibangan |
-| `-pa` | `-ng` | 55 words | pagsabing, paggurang |
-| `-ma` | `-ng` | 52 words | magarang, matabang |
+| `-pi` | `-n` | 77 words | pinagkukuanan, pinagkakaputan |
+| `-pa` | `-n` | 75 words | paluan, painiton |
+| `-ka` | `-n` | 75 words | kakagaton, katangaan |
+| `-na` | `-a` | 74 words | nagbabareta, nagsaranga |
+| `-pi` | `-an` | 72 words | pinagkukuanan, pinagkakaputan |
+| `-pa` | `-a` | 67 words | pamareta, padilla |
+| `-ka` | `-an` | 67 words | katangaan, kagadanan |
+| `-na` | `-n` | 66 words | naiisihan, nahaman |
+| `-ma` | `-a` | 64 words | manusela, mababareta |
+| `-na` | `-an` | 56 words | naiisihan, nahaman |
 
 ### 6.5 Recursive Morpheme Segmentation
 
@@ -486,26 +522,26 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
-| pagpapamahalang | **`pag-pa-pa-ma-hala-ng`** | 10.5 | `hala` |
-| pinakagurangan | **`pi-na-ka-gura-ng-an`** | 10.5 | `gura` |
-| pinakaprimerang | **`pi-na-ka-primera-ng`** | 9.0 | `primera` |
-| nakakapaugma | **`na-ka-ka-pa-ugma`** | 9.0 | `ugma` |
-| nakapagpalupad | **`na-ka-pag-pa-lupad`** | 9.0 | `lupad` |
-| makatarungan | **`ma-ka-taru-ng-an`** | 9.0 | `taru` |
-| nakakasumo | **`na-ka-ka-sumo`** | 7.5 | `sumo` |
-| pagpapainit | **`pag-pa-pa-init`** | 7.5 | `init` |
-| nagpapalihis | **`nag-pa-pa-lihis`** | 7.5 | `lihis` |
-| pagkanamamanwaan | **`pag-ka-na-ma-ma-nwaan`** | 7.5 | `nwaan` |
-| nagpabistong | **`nag-pa-bist-ong`** | 7.5 | `bist` |
-| nakakalangkaw | **`na-ka-ka-langkaw`** | 7.5 | `langkaw` |
-| nagpapalibot | **`nag-pa-pa-libot`** | 7.5 | `libot` |
-| makakapugol | **`ma-ka-ka-pugol`** | 7.5 | `pugol` |
-| pagkitabangan | **`pag-kitaba-ng-an`** | 7.5 | `kitaba` |
+| pinakamalumoy | **`pi-na-ka-ma-lumoy`** | 9.0 | `lumoy` |
+| pinakamakosog | **`pi-na-ka-ma-kosog`** | 9.0 | `kosog` |
+| pinakagrabeng | **`pi-na-ka-grabe-ng`** | 9.0 | `grabe` |
+| pinakaposibleng | **`pi-na-ka-posible-ng`** | 9.0 | `posible` |
+| pinakadarakula | **`pi-na-ka-darakula`** | 7.5 | `darakula` |
+| pagpapasakit | **`pag-pa-pa-sakit`** | 7.5 | `sakit` |
+| nakakasakop | **`na-ka-ka-sakop`** | 7.5 | `sakop` |
+| nakakahimo | **`na-ka-ka-himo`** | 7.5 | `himo` |
+| pinakasikat | **`pi-na-ka-sikat`** | 7.5 | `sikat` |
+| nakakalihis | **`na-ka-ka-lihis`** | 7.5 | `lihis` |
+| pagkakamukna | **`pag-ka-ka-mukna`** | 7.5 | `mukna` |
+| nagpapaluwas | **`nag-pa-pa-luwas`** | 7.5 | `luwas` |
+| pinakaligtas | **`pi-na-ka-ligtas`** | 7.5 | `ligtas` |
+| nagpapamidbid | **`nag-pa-pa-midbid`** | 7.5 | `midbid` |
+| nakakalayog | **`na-ka-ka-layog`** | 7.5 | `layog` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language BCL appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Central Bikol shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
 
 ---
 ## 7. Summary & Recommendations
@@ -517,8 +553,8 @@ The language BCL appears to be more isolating or has a highly fixed vocabulary.
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
 | Tokenizer | **64k BPE** | Best compression (4.81x) |
-| N-gram | **2-gram** | Lowest perplexity (216) |
-| Markov | **Context-4** | Highest predictability (95.1%) |
+| N-gram | **2-gram** | Lowest perplexity (215) |
+| Markov | **Context-4** | Highest predictability (95.0%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
 
@@ -732,4 +768,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 06:41:18*
+*Report Date: 2026-01-03 18:57:54*

models/embeddings/aligned/bcl_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f400c99cab465c72ec8b6062187b41e9c52f4f95640b96cf878017c5cea209fc
+size 1102108405

models/embeddings/aligned/bcl_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "bcl", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bcl_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dbf43b1ef3b842abc044927093a568ddcfa17a12d1797e5594792088b75799c
+size 65664

models/embeddings/aligned/bcl_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bcl",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 37817,
+  "vocab_size": 75002
+}

models/embeddings/aligned/bcl_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59b895badc838a29b59876852a136c4a56a05b518d24994516fcf0f9b9665497
+size 276506869

models/embeddings/aligned/bcl_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "bcl", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bcl_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c936c946219d2f2650d716a096747470558b9e896f0bd3bd6549cba676ffa0a
+size 4224

models/embeddings/aligned/bcl_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bcl",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 37817,
+  "vocab_size": 75002
+}

models/embeddings/aligned/bcl_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e9c20235632b2c67d33f72b8910f4b4f3805205d01fabf11a992c84cf8c7a97
+size 551707381

models/embeddings/aligned/bcl_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "bcl", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bcl_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2866fdccb6a9036352c833a80b97ee8bd6a63c69bfbcb4525f4d3954a840659c
+size 16512

models/embeddings/aligned/bcl_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bcl",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 37817,
+  "vocab_size": 75002
+}

models/embeddings/monolingual/bcl_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf3cdb55fbf130648cf3ffee8edc3e5ecdf0fb3e434c5eb95017d070b6ed68d5
-size 1101739857
+oid sha256:f400c99cab465c72ec8b6062187b41e9c52f4f95640b96cf878017c5cea209fc
+size 1102108405

models/embeddings/monolingual/bcl_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 74648
+  "vocab_size": 75002
 }

models/embeddings/monolingual/bcl_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc294724d9b9b5f9e9882106bbd52a196e0d3ff5d980d094bd1552d8c0d1f5e1
-size 276410193
+oid sha256:59b895badc838a29b59876852a136c4a56a05b518d24994516fcf0f9b9665497
+size 276506869

models/embeddings/monolingual/bcl_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 74648
+  "vocab_size": 75002
 }

models/embeddings/monolingual/bcl_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:62e5f9c945bbfd63c2c974b8baef649c3879ac74b839bc579143094844889c04
-size 551520081
+oid sha256:8e9c20235632b2c67d33f72b8910f4b4f3805205d01fabf11a992c84cf8c7a97
+size 551707381

models/embeddings/monolingual/bcl_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 74648
+  "vocab_size": 75002
 }

models/subword_markov/bcl_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:006988c9f7d26c290cee7f3efcec68f438194198095ef615a657d959d50bffe5
-size 281190
+oid sha256:3b9f7f23d300f937e4790bb766184ca347c4decb9122fd47208c197396a989c6
+size 282765

models/subword_markov/bcl_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "bcl",
-  "unique_contexts": 7079,
-  "total_transitions": 38302882
+  "unique_contexts": 7145,
+  "total_transitions": 38644229
 }

models/subword_markov/bcl_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae7603efc10031eeb84e9f4ea835d551f816490109984087110f408b59dbe37e
-size 1134736
+oid sha256:6018a3db4789319859319dd16abd0f5206f74e4175e76c7036275c2d110ff691
+size 1167741

models/subword_markov/bcl_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "bcl",
-  "unique_contexts": 38137,
-  "total_transitions": 38281621
+  "unique_contexts": 38469,
+  "total_transitions": 38622772
 }

models/subword_markov/bcl_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49ee3d03628a769fa0eaa6565a8c6cb451eeabe76d6105b458158c2fe59d0663
-size 3801377
+oid sha256:d3b3338179235aefbbad0c41749adf4f11a37961c86959ff618b47a2dadef047
+size 3820884

models/subword_markov/bcl_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "bcl",
-  "unique_contexts": 128219,
-  "total_transitions": 38260360
+  "unique_contexts": 128967,
+  "total_transitions": 38601315
 }

models/subword_markov/bcl_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff2037012deaf809cbae28b5f83dbde59b2bd458ca4152e6a4cb0aacfe9c3446
-size 11339203
+oid sha256:cc6061d9e18cc7035f85cd628a3f996d6455bad0245b42827e1039d29342814d
+size 11425249

models/subword_markov/bcl_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "bcl",
-  "unique_contexts": 463318,
-  "total_transitions": 38239099
+  "unique_contexts": 465409,
+  "total_transitions": 38579858
 }

models/subword_ngram/bcl_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:99ef93eead11f74c9add74db6169310c53797885fcb53f36d200d66ee5ef38b7
-size 89797
+oid sha256:57709d0022922b843993ba5817848b7f294d115a839726bd9488723c042a386b
+size 90307

models/subword_ngram/bcl_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "bcl",
-  "unique_ngrams": 6792,
-  "total_ngrams": 38302882
+  "unique_ngrams": 6829,
+  "total_ngrams": 38644229
 }

models/subword_ngram/bcl_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:60182df9a558595b603a59b59d1c3ec77041802aa43a6afa665a1b221f701b27
-size 597753
+oid sha256:8ea7a7402351c828dbe6f03ae90eee5a8c1ab48582a1470d25cabd58f2ba4b3d
+size 603314

models/subword_ngram/bcl_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "bcl",
-  "unique_ngrams": 46201,
-  "total_ngrams": 38281621
+  "unique_ngrams": 46307,
+  "total_ngrams": 38622772
 }

models/subword_ngram/bcl_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:345f2173509b2b9bc8eed2c825e6591508bf37f060bb0ad7b1e85c6fc662db4c
-size 2804511
+oid sha256:1c01f0e5fa98c97bc8af7f4154f9dd31c030e10599a5c30afc083c08fbeab311
+size 2812697

models/subword_ngram/bcl_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "bcl",
-  "unique_ngrams": 248296,
-  "total_ngrams": 38260360
+  "unique_ngrams": 249114,
+  "total_ngrams": 38601315
 }

models/subword_ngram/bcl_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e420fed88b6ae6c6706879701bf0cf57a237d4d5619e3bf1df3a27bd8576a2a
+size 8149157

models/subword_ngram/bcl_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "bcl",
+  "unique_ngrams": 711663,
+  "total_ngrams": 38579858
+}

models/tokenizer/bcl_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:77a1b85ce56e78c7903b6f9dea869cefef456654bcf07353809aacdd461381f0
-size 505737
+oid sha256:adc85c4a3f7a5ea359a7ee830bc0867e4e0c8a908a7d7a5a592552fef48dbaa0
+size 506019

models/tokenizer/bcl_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a387469a476988f3e12c7fd375505ee9b078c2d19b0076337812e87e50d6f204
-size 782150
+oid sha256:291a63acf5a40cd7b78b3bf2affbaefc5876b6c279f7aa29b65d942564818dd1
+size 781718

models/tokenizer/bcl_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0354558806559d124b6246dfc4d681e614984d19649abc6f6c225ad6e3692aae
-size 1348303
+oid sha256:87f1822c949c125c290743a780dafb6dc53e13d950da945bfed552334d7f6f53
+size 1347091

models/tokenizer/bcl_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:94f252a579c2ba992311623e32fbb194c272d1a66be40c17c4ba4f8a8a434b14
-size 370855
+oid sha256:ee3b3681497a067e58037888c88caebf336f2a54fc979d0abe3c83393bb50d49
+size 370844

models/vocabulary/bcl_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:261d873870e07e5f3af5f7a71946694964dd00bc89c0db0799bfdffcdcae911b
-size 2250849
+oid sha256:ee709c7a643dfce2255dbff02ca689c28dc8038d6a3e21161b5bf86b201047c3
+size 2257602

models/vocabulary/bcl_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "bcl",
-  "vocabulary_size": 131763,
+  "vocabulary_size": 132282,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.05388651103925395,
+    "type_token_ratio": 0.05366677708174099,
     "coverage": {
-      "top_100": 0.41810468235658227,
-      "top_1000": 0.6157275307187713,
-      "top_5000": 0.7676151406101507,
-      "top_10000": 0.8261024576825995
+      "top_100": 0.41912867027609363,
+      "top_1000": 0.6163627971043987,
+      "top_5000": 0.7681565315350177,
+      "top_10000": 0.8265210273913567
     },
-    "hapax_count": 195915,
-    "hapax_ratio": 0.5978887810594548,
-    "total_documents": 21261
+    "hapax_count": 197095,
+    "hapax_ratio": 0.5983872583695886,
+    "total_documents": 21457
   }
 }

models/word_markov/bcl_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6680aaaf3a96d97340daa50f62f0e696137d5ec1eb9abc2d5323a1ed5594af5e
-size 18651232
+oid sha256:811fbf71c80d2bc116560b97aa5b380f3ac9646b50219ef7a3470f7fed8ec4eb
+size 18870229

models/word_markov/bcl_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "bcl",
-  "unique_contexts": 327423,
-  "total_transitions": 6059630
+  "unique_contexts": 329127,
+  "total_transitions": 6115990
 }

models/word_markov/bcl_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c5f462dad3387aa8f577ce6600d0c3d3137f0901270876d01c03c667bd229db
-size 47432243
+oid sha256:8e02c6ad66efe38a58e3f764b9e72570bd799d093449cb92d4e8defac8561823
+size 47619941

models/word_markov/bcl_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "bcl",
-  "unique_contexts": 2054215,
-  "total_transitions": 6038369
+  "unique_contexts": 2064138,
+  "total_transitions": 6094533
 }