Upload all models and assets for bo (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: bo
-language_name: BO
+language_name: Tibetan
 language_family: tibetoburman_tibetic
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-tibetoburman_tibetic
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 5.300
+    value: 5.306
   - name: best_isotropy
     type: isotropy
-    value: 0.8494
+    value: 0.8542
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# BO - Wikilangs Models
+# Tibetan - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **BO** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Tibetan** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 4.065x | 4.07 | 0.3680% | 234,234 |
-| **16k** | 4.561x | 4.56 | 0.4129% | 208,750 |
-| **32k** | 4.981x | 4.98 | 0.4510% | 191,137 |
-| **64k** | 5.300x 🏆 | 5.30 | 0.4798% | 179,650 |
+| **8k** | 4.069x | 4.07 | 0.3678% | 233,845 |
+| **16k** | 4.567x | 4.57 | 0.4127% | 208,371 |
+| **32k** | 4.989x | 4.99 | 0.4509% | 190,738 |
+| **64k** | 5.306x 🏆 | 5.31 | 0.4795% | 179,358 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `ཞེ་ཆེན་དགོན་ནི་ཞེ་ཆེན་རབ་འབྱམས་དང་པོ་བསྟན་པའི་རྒྱལ་མཚན་གྱིས་ཕྱག་བཏབ་པ་ཡིན།`
+**Sample 1:** `གསེར་མོ་ནི་སྒོང་སྐྱེས་སྲོག་ཆགས་ཀྱི་རིགས་གཅིག་རེད། ལོ་རྒྱུས། པར་རིས་བར་འཁྱམས། ཟིན...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁ཞེ་ ཆེན་ དགོན་ ནི་ ཞེ་ ཆེན་ རབ་འབྱམས་ དང་པོ་ བསྟན་པའི་ རྒྱལ་མཚན་ ... (+3 more)` | 13 |
-| 16k | `▁ཞེ་ ཆེན་ དགོན་ནི་ ཞེ་ ཆེན་ རབ་འབྱམས་ དང་པོ་ བསྟན་པའི་ རྒྱལ་མཚན་གྱིས་ ཕྱག་བཏབ་ ... (+1 more)` | 11 |
-| 32k | `▁ཞེ་ཆེན་ དགོན་ནི་ ཞེ་ཆེན་ རབ་འབྱམས་ དང་པོ་ བསྟན་པའི་ རྒྱལ་མཚན་གྱིས་ ཕྱག་བཏབ་ པ་ཡིན།` | 9 |
-| 64k | `▁ཞེ་ཆེན་ དགོན་ནི་ ཞེ་ཆེན་ རབ་འབྱམས་ དང་པོ་ བསྟན་པའི་ རྒྱལ་མཚན་གྱིས་ ཕྱག་བཏབ་ པ་ཡིན།` | 9 |
+| 8k | `▁གསེར་ མོ་ནི་ སྒོང་སྐྱེས་ སྲོག་ཆགས་ཀྱི་ རིགས་གཅིག་རེད། ▁ལོ་རྒྱུས། ▁པར་རིས་བར་ འཁྱམས། ▁ཟིན་ཐོ་ འམ་དཔྱད་གཞི། ... (+5 more)` | 15 |
+| 16k | `▁གསེར་ མོ་ནི་ སྒོང་སྐྱེས་ སྲོག་ཆགས་ཀྱི་ རིགས་གཅིག་རེད། ▁ལོ་རྒྱུས། ▁པར་རིས་བར་ འཁྱམས། ▁ཟིན་ཐོ་ འམ་དཔྱད་གཞི། ... (+5 more)` | 15 |
+| 32k | `▁གསེར་ མོ་ནི་ སྒོང་སྐྱེས་ སྲོག་ཆགས་ཀྱི་ རིགས་གཅིག་རེད། ▁ལོ་རྒྱུས། ▁པར���རིས་བར་ འཁྱམས། ▁ཟིན་ཐོ་ འམ་དཔྱད་གཞི། ... (+5 more)` | 15 |
+| 64k | `▁གསེར་ མོ་ནི་ སྒོང་སྐྱེས་ སྲོག་ཆགས་ཀྱི་ རིགས་གཅིག་རེད། ▁ལོ་རྒྱུས། ▁པར་རིས་བར་ འཁྱམས། ▁ཟིན་ཐོ་ འམ་དཔྱད་གཞི། ... (+5 more)` | 15 |
 
-**Sample 2:** `རང་གི་ཕ་མའི་ཁྱིམ་དུ་འཚོ་བ་སྐྱེལ་བའི་ཞེ་སའི་ཚིག ༼དུང་དཀར་ཚིག་མཛོད་ཆེན་མོ་༽ནས་བཏུས...`
+**Sample 2:** `ཀྲོའུ་སི། ཞི་ལའི་ལྷ་སྒྲུང་ཁྲོད་ཀྱི་ལྷ་རེད། མི་ཚེ། པར་རིས་བར་འཁྱམས། ཟིན་ཐོ་འམ་དཔྱ...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁རང་གི་ ཕ ་མའི་ ཁྱིམ་ དུ་ འཚོ་བ་ སྐྱ ེལ་བའི་ ཞེ་ སའི་ ... (+4 more)` | 14 |
-| 16k | `▁རང་གི་ ཕ་མའི་ ཁྱིམ་དུ་ འཚོ་བ་ སྐྱ ེལ་བའི་ ཞེ་སའི་ ཚིག ▁༼དུང་ དཀར་ཚིག་མཛོད་ ... (+1 more)` | 11 |
-| 32k | `▁རང་གི་ ཕ་མའི་ ཁྱིམ་དུ་ འཚོ་བ་ སྐྱེལ་བའི་ ཞེ་སའི་ཚིག ▁༼དུང་ དཀར་ཚིག་མཛོད་ ཆེན་མོ་༽ནས་བཏུས།` | 9 |
-| 64k | `▁རང་གི་ཕ་མའི་ ཁྱིམ་དུ་ འཚོ་བ་སྐྱེལ་བའི་ ཞེ་སའི་ཚིག ▁༼དུང་ དཀར་ཚིག་མཛོད་ ཆེན་མོ་༽ནས་བཏུས།` | 7 |
+| 8k | `▁ཀྲ ོའུ་ སི། ▁ཞི་ ལའི་ ལྷ་ སྒྲུང་ ཁྲོད་ཀྱི་ ལྷ་ རེད། ... (+10 more)` | 20 |
+| 16k | `▁ཀྲོའུ་ སི། ▁ཞི་ ལའི་ ལྷ་སྒྲུང་ ཁྲོད་ཀྱི་ ལྷ་རེད། ▁མི་ཚེ། ▁པར་རིས་བར་ འཁྱམས། ... (+7 more)` | 17 |
+| 32k | `▁ཀྲོའུ་ སི། ▁ཞི་ ལའི་ ལྷ་སྒྲུང་ ཁྲོད་ཀྱི་ ལྷ་རེད། ▁མི་ཚེ། ▁པར་རིས་བར་ འཁྱམས། ... (+7 more)` | 17 |
+| 64k | `▁ཀྲོའུ་ སི། ▁ཞི་ ལའི་ ལྷ་སྒྲུང་ ཁྲོད་ཀྱི་ ལྷ་རེད། ▁མི་ཚེ། ▁པར་རིས་བར་ འཁྱམས། ... (+7 more)` | 17 |
 
-**Sample 3:** `དུས་རྟག་ཏུ་སྐྱེ་འཇིག་མི་བྱེད་པ། དཔེར་ན། ནམ་མཁའ་ལྟ་བུ།`
+**Sample 3:** `མྱང་འདས་གཞན་ནས་སྒྲུབ་ཏུ་མེད། མྱ་ངན་ལས་འདས་པ་སྟེ་ཐར་པ་དང་། ཐམས་ཅད་མཁྱེན་པའི་གོ་འཕ...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁དུས་ རྟག་ཏུ་ སྐྱེ་ འཇིག་ མི་ བྱེད་པ། ▁དཔེར་ན། ▁ནམ་མཁའ་ ལྟ་བུ།` | 9 |
-| 16k | `▁དུས་ རྟག་ཏུ་ སྐྱེ་ འཇིག་ མི་ བྱེད་པ། ▁དཔེར་ན། ▁ནམ་མཁའ་ ལྟ་བུ།` | 9 |
-| 32k | `▁དུས་ རྟག་ཏུ་ སྐྱེ་འཇིག་ མི་ བྱེད་པ། ▁དཔེར་ན། ▁ནམ་མཁའ་ ལྟ་བུ།` | 8 |
-| 64k | `▁དུས་རྟག་ཏུ་ སྐྱེ་འཇིག་ མི་བྱེད་པ། ▁དཔེར་ན། ▁ནམ་མཁའ་ ལྟ་བུ།` | 6 |
+| 8k | `▁མྱང་ འདས་ གཞན་ ནས་ སྒྲུབ་ ཏུ་ མེད། ▁མྱ་ངན་ ལས་འདས་ པ་སྟེ་ ... (+15 more)` | 25 |
+| 16k | `▁མྱང་འདས་ གཞན་ ནས་ སྒྲུབ་ ཏུ་ མེད། ▁མྱ་ངན་ ལས་འདས་ པ་སྟེ་ ཐར་ ... (+13 more)` | 23 |
+| 32k | `▁མྱང་འདས་ གཞན་ནས་ སྒྲུབ་ ཏུ་ མེད། ▁མྱ་ངན་ ལས་འདས་ པ་སྟེ་ ཐར་ པ་དང་། ... (+10 more)` | 20 |
+| 64k | `▁མྱང་འདས་ གཞན་ནས་ སྒྲུབ་ ཏུ་ མེད། ▁མྱ་ངན་ལས་འདས་ པ་སྟེ་ ཐར་ པ་དང་། ▁ཐམས་ཅད་ ... (+7 more)` | 17 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 5.300x compression
-- **Lowest UNK Rate:** 8k with 0.3680% unknown tokens
+- **Best Compression:** 64k achieves 5.306x compression
+- **Lowest UNK Rate:** 8k with 0.3678% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 36,069 | 15.14 | 160,381 | 8.0% | 26.4% |
-| **2-gram** | Subword | 469 🏆 | 8.87 | 14,734 | 57.9% | 90.6% |
-| **3-gram** | Word | 207,650 | 17.66 | 482,234 | 3.8% | 11.0% |
-| **3-gram** | Subword | 3,733 | 11.87 | 86,351 | 25.0% | 62.7% |
-| **4-gram** | Word | 569,587 | 19.12 | 997,749 | 3.3% | 7.5% |
-| **4-gram** | Subword | 21,504 | 14.39 | 391,088 | 12.0% | 36.1% |
+| **2-gram** | Word | 35,575 | 15.12 | 163,426 | 8.0% | 26.6% |
+| **2-gram** | Subword | 468 🏆 | 8.87 | 14,902 | 58.0% | 90.7% |
+| **3-gram** | Word | 208,497 | 17.67 | 499,603 | 3.7% | 11.0% |
+| **3-gram** | Subword | 3,697 | 11.85 | 87,521 | 25.1% | 62.9% |
+| **4-gram** | Word | 574,996 | 19.13 | 1,035,818 | 3.2% | 7.7% |
+| **4-gram** | Subword | 21,129 | 14.37 | 395,961 | 12.1% | 36.3% |
+| **5-gram** | Word | 554,814 | 19.08 | 896,814 | 3.6% | 8.0% |
+| **5-gram** | Subword | 85,765 | 16.39 | 872,546 | 6.0% | 20.2% |
 
 ### Top 5 N-grams by Size
 
@@ -150,68 +162,88 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `པ དང` | 26,663 |
-| 2 | `པ ལ` | 12,165 |
-| 3 | `བ དང` | 12,147 |
-| 4 | `ཐམས ཅད` | 11,625 |
-| 5 | `པ ནི` | 10,955 |
+| 1 | `པ དང` | 28,306 |
+| 2 | `བ དང` | 12,858 |
+| 3 | `པ ལ` | 12,495 |
+| 4 | `ཐམས ཅད` | 12,121 |
+| 5 | `པ ནི` | 11,602 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `སྤྱོད འཇུག གི` | 4,095 |
-| 2 | `ད དུང གཟིགས` | 3,422 |
-| 3 | `ཞེས བྱ བ` | 3,401 |
-| 4 | `ཕྱི ཕྱོགས དྲ` | 3,394 |
-| 5 | `ཕྱོགས དྲ མཐུད` | 3,394 |
+| 1 | `སྤྱོད འཇུག གི` | 4,094 |
+| 2 | `ཞེས བྱ བ` | 3,742 |
+| 3 | `ད དུང གཟིགས` | 3,594 |
+| 4 | `ཕྱོགས དྲ མཐུད` | 3,563 |
+| 5 | `ཕྱི ཕྱོགས དྲ` | 3,563 |
 
 **4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `ཕྱི ཕྱོགས དྲ མཐུད` | 3,393 |
+| 1 | `ཕྱི ཕྱོགས དྲ མཐུད` | 3,562 |
 | 2 | `དཔྱད གཞིའི དཀར ཆག` | 3,391 |
 | 3 | `ཟིན ཐོ འམ དཔྱད` | 2,805 |
 | 4 | `ཐོ འམ དཔྱད གཞི` | 2,802 |
-| 5 | `ད དུང གཟིགས ཕྱི` | 2,789 |
+| 5 | `དུང གཟིགས ཕྱི ཕྱོགས` | 2,789 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `ཟིན ཐོ འམ དཔྱད གཞི` | 2,802 |
+| 2 | `ད དུང གཟིགས ཕྱི ཕྱོགས` | 2,789 |
+| 3 | `གཟིགས ཕྱི ཕྱོགས དྲ མཐུད` | 2,779 |
+| 4 | `དཀར ཆག ད དུང གཟིགས` | 2,777 |
+| 5 | `དཔྱད གཞིའི དཀར ཆག ད` | 2,776 |
 
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `ས ་` | 1,063,653 |
-| 2 | `། _` | 775,741 |
-| 3 | `ང ་` | 696,058 |
-| 4 | `ན ་` | 582,326 |
-| 5 | `་ བ` | 571,331 |
+| 1 | `ས ་` | 1,109,782 |
+| 2 | `། _` | 814,181 |
+| 3 | `ང ་` | 726,970 |
+| 4 | `ན ་` | 605,125 |
+| 5 | `་ བ` | 601,943 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `་ པ ་` | 222,811 |
-| 2 | `ག ས ་` | 205,307 |
-| 3 | `། _ །` | 180,441 |
-| 4 | `ས ་ པ` | 161,245 |
-| 5 | `་ ད ང` | 151,339 |
+| 1 | `་ པ ་` | 233,799 |
+| 2 | `ག ས ་` | 214,635 |
+| 3 | `། _ །` | 181,451 |
+| 4 | `ས ་ པ` | 169,152 |
+| 5 | `་ ད ང` | 160,512 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `་ ད ང ་` | 128,716 |
-| 2 | `་ པ འི ་` | 107,145 |
-| 3 | `ང ་ ། _` | 84,367 |
-| 4 | `ས ་ པ ་` | 74,777 |
-| 5 | `་ པ ར ་` | 62,788 |
+| 1 | `་ ད ང ་` | 137,863 |
+| 2 | `་ པ འི ་` | 114,983 |
+| 3 | `ང ་ ། _` | 88,853 |
+| 4 | `ས ་ པ ་` | 77,821 |
+| 5 | `་ པ ར ་` | 67,023 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `ད ང ་ ། _` | 50,908 |
+| 2 | `་ ད ང ་ །` | 50,893 |
+| 3 | `ས ་ པ འི ་` | 39,175 |
+| 4 | `་ རྣ མ ས ་` | 29,571 |
+| 5 | `་ སོ ག ས ་` | 28,140 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram (subword) with 469
+- **Best Perplexity:** 2-gram (subword) with 468
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~36% of corpus
+- **Coverage:** Top-1000 patterns cover ~20% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.9258 | 1.900 | 17.80 | 44,551 | 7.4% |
-| **1** | Subword | 0.8301 | 1.778 | 6.82 | 8,378 | 17.0% |
-| **2** | Word | 0.7004 | 1.625 | 3.77 | 792,341 | 30.0% |
-| **2** | Subword | 0.4672 | 1.382 | 4.10 | 57,113 | 53.3% |
-| **3** | Word | 0.2866 | 1.220 | 1.60 | 2,987,004 | 71.3% |
-| **3** | Subword | 0.4482 | 1.364 | 3.28 | 233,848 | 55.2% |
-| **4** | Word | 0.1070 🏆 | 1.077 | 1.17 | 4,767,837 | 89.3% |
-| **4** | Subword | 0.3731 | 1.295 | 2.37 | 765,950 | 62.7% |
+| **1** | Word | 0.9206 | 1.893 | 17.76 | 45,103 | 7.9% |
+| **1** | Subword | 0.8281 | 1.775 | 6.83 | 8,393 | 17.2% |
+| **2** | Word | 0.7033 | 1.628 | 3.81 | 800,524 | 29.7% |
+| **2** | Subword | 0.4670 | 1.382 | 4.11 | 57,328 | 53.3% |
+| **3** | Word | 0.2921 | 1.224 | 1.62 | 3,051,550 | 70.8% |
+| **3** | Subword | 0.4481 | 1.364 | 3.28 | 235,662 | 55.2% |
+| **4** | Word | 0.1112 🏆 | 1.080 | 1.18 | 4,929,019 | 88.9% |
+| **4** | Subword | 0.3733 | 1.295 | 2.38 | 773,603 | 62.7% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `པ ཐམས ཅད དུ བཅུག དེ བསིལ ཟེར མི བཏུབ པའི ཚུལ ཁྲིམས རྒྱལ པོ ཆེའི`
-2. `དང ཁ སྐོང ཝེར ཐུགས རྗེ གླུའི བསྟོད ས ལ བླ མའི སྲས འཇུག གི སྐད`
-3. `ལ བཙུན བསྟེན 2 5 ལོ རྒྱུས ས བཅད རྩོམ རིག སྔགས སོ ཁོས ང མ`
+1. `པ ཡིད གཉིས གདན ཤོ མིག གསུམ རྫིང བུར ལན གསུམ པ ལ མཱུ ཥི ཏ`
+2. `དང སུམ གཉིས ཀྱི ཞབས ལྕགས རིགས སུ བཞུགས པ ཆེན མོ གཉིས ཀྱི ཚད ལས`
+3. `ལ བོད ཤན ནམ ཞིག ལུས ལ སོགས པའི མིང པེལ རེས མོས བརྡུང བའི སྐུ`
 
 **Context Size 2:**
 
-1. `པ དང ཉེ བའི མཆོད སྤྲིན རྣམ སྡུད དང ཞུས ཤིག བྱས བྱུང ༣༩ སྤྲིན པ བར`
-2. `པ ལ དབང ཐོབ ཤོག ཤུ དག ལི ཁྲིའི ཚོན གྱིས རབ མཛེས ཁ བའི གོས བཟང`
-3. `བ དང གཅོག པར བྱེད པ ཙམ ལ འཁྲུལ མིན རྨི སོགས རྣམ དབྱེ བཞི བའི དེ`
+1. `པ དང གདམ ང མ གཉིས གཉིས གཉིས ཡོད ཤར ཕོགས ཀི པཎི ཏ ཨ བྷི ཥིཉྩ`
+2. `བ དང མནའ སྐྱེལ ཞིང དབྱར ཀ རི ཀ སྤྱི མཐུན རྒྱལ ཁབ དེ རུ བཞག གོ`
+3. `པ ལ བཞུགས པར ཞལ གྱིས བཞེས པ ནས འབྲས བུ ཉེ ཟ དེའི བྱེ བྲག པ`
 
 **Context Size 3:**
 
-1. `སྤྱོད འཇུག གི རྣམ བཤད ཐེག ཆེན ཆོས ཀྱི རྒྱ མཚོ སོགས བྱོན རྒྱ དཀར ནག སེར དང`
-2. `ད དུང གཟིགས ཕྱི ཕྱོགས དྲ མཐུད དབྱིན ཇིའི རླུང འཕྲིན ཀུང སིས ཉིན དེར ཉིའུ གཡོད སྐབས`
-3. `ཞེས བྱ བ བསྲུང བའི དམ ཚིག ཅན ཐུགས དམ སྐུལ བ ནི འདི ལ རྟགས ངེས པ`
+1. `སྤྱོད འཇུག གི དཀའ འགྲེལ ཤིང དཔར ཞེས གསུངས པ ནི འདོད པ ཁྱབ ཁོངས ཡངས པ དེ`
+2. `ཞེས བྱ བ ལ སོགས པ གཞན མ ཡིན ནོ རབ འབར དགྲ ཡི དབང དུ ཟད འཕེལ`
+3. `ད དུང གཟིགས ཕྱི ཕྱོགས དྲ མཐུད ལྕེ དཔྱད གཞིའི དཀར ཆག ད དུང གཟིགས ཀྱེ རྡོ རྗེ`
 
 **Context Size 4:**
 
-1. `དཔྱད གཞིའི དཀར ཆག ད དུང གཟིགས བཙན པོ རིམ བྱོན གྱི མཚན བྱང བྲག གདོང བཀྲས གླིང དབང`
-2. `ཟིན ཐོ འམ དཔྱད གཞི དཔྱད གཞིའི དཀར ��ག ད དུང གཟིགས ཕྱི ཕྱོགས དྲ མཐུད ཤིང འཛུགས དུས`
-3. `ཐོ འམ དཔྱད གཞི དཔྱད གཞིའི དཀར ཆག ད དུང གཟིགས ཕྱི ཕྱོགས དྲ མཐུད དབྱིན ཇིའི རླུང འཕྲིན`
+1. `དཔྱད གཞིའི དཀར ཆག ད དུང གཟིགས ཕྱི ཕྱོགས དྲ མཐུད དབྱིན ཇིའི རླུང འཕྲིན ཀུང སིས ཉིན དེར`
+2. `ཟིན ཐོ འམ དཔྱད གཞི དཔྱད གཞིའི དཀར ཆག ད དུང གཟིགས ཕྱི ཕྱོགས དྲ མཐུད bdrc buddhist digital`
+3. `ཐོ འམ དཔྱད གཞི དཔྱད གཞིའི དཀར ཆག ད དུང གཟིགས གེ སར རྒྱལ པོ རྒྱ ནག ཏུ ཕེབས`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `་ཁྲགཟུང་སྒྲིག་ཐུགནསལ་གྱུ`
-2. `ས་ན་པོ།_ལྷན་ཚད་གན་`
-3. `གཞིགས་བ་དྷྱ་བས་སྦྱིན།_`
+1. `་དུ་མོ་སྦྱངས་མསལ་ཁྲི་ཁ`
+2. `ས་ཉིས།_ཞནང་ཀྱིས།_དང`
+3. `གས་ཆེན་ནི་ཡོད་འ༔_།_`
 
 **Context Size 2:**
 
-1. `ས་ལ་པའི་ཐུགས་ནག་འབྱོར`
-2. `།_རབས་ཀྱིས་དཀྱིལ་སྣང་གི`
-3. `ང་མ་བུདྡྷ་ཀྵེ་ཙཱ་བར་ནས།`
+1. `ས་དངོས་ཀྱི་ལྷས་ད་ེ_རྣམས`
+2. `།_ཁེངས་བཞིན་པའི་སྡུག་ཡི`
+3. `ང་།_ད་གཉིས།_དྲངས་པར`
 
 **Context Size 3:**
 
-1. `་པ་སྟེ།_ལྷ་ལྡན་ཡོང་ཡེ་ཤེས`
-2. `གས་མད་ོ_འགྱུར་རྡོ་རྗེའི་བསྲུ`
-3. `།_།རྣམ་འཆོར་བ་དང་སེམས`
+1. `་པ་ལ་དྲིས་ན་ནི་_ལོའི་རྒྱུད`
+2. `གས་པ་གླིང་བཙན་པོ་དབྱིངས`
+3. `།_།བྱས་ཀྱང་རུང་།_མི་དང`
 
 **Context Size 4:**
 
-1. `་དང་ཕྱོགས་སུ་བཞིན་ཉུལ་ཆོས`
-2. `་པའི་སེམས་སྐྱེས་ཆེན་ཨུ་ཡོན་`
-3. `ང་།_མདོ་སྡུད་པ་ཅན་ཟན་རི`
+1. `་དང་།_།མཐུ་སྟོབས་རྒྱས་མཛེ`
+2. `་པའི་ཚུལ་བཻ་སེར་པོ་འཁོར་ཡ`
+3. `ང་།_བཞི་པ།_སྟོན་པ་སྒྲུབ་པ`
 
 
 ### Key Findings
 
-- **Best Predictability:** Context-4 (word) with 89.3% predictability
+- **Best Predictability:** Context-4 (word) with 88.9% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (765,950 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (773,603 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,64 +346,64 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 18,720 |
-| Total Tokens | 7,245,735 |
-| Mean Frequency | 387.06 |
+| Vocabulary Size | 18,977 |
+| Total Tokens | 7,591,805 |
+| Mean Frequency | 400.05 |
 | Median Frequency | 5 |
-| Frequency Std Dev | 3716.05 |
+| Frequency Std Dev | 3886.00 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | པ | 262,584 |
-| 2 | དང | 156,471 |
-| 3 | ལ | 145,900 |
-| 4 | བ | 121,705 |
-| 5 | པའི | 110,790 |
-| 6 | མ | 88,147 |
-| 7 | དེ | 78,304 |
-| 8 | ནི | 74,845 |
-| 9 | ཀྱི | 70,464 |
-| 10 | དུ | 70,132 |
+| 1 | པ | 277,831 |
+| 2 | དང | 165,810 |
+| 3 | ལ | 150,300 |
+| 4 | བ | 127,823 |
+| 5 | པའི | 118,705 |
+| 6 | མ | 92,873 |
+| 7 | དེ | 80,387 |
+| 8 | ནི | 78,884 |
+| 9 | ཀྱི | 76,665 |
+| 10 | དུ | 73,981 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | པིཎྜཱརྠ | 2 |
-| 2 | saṃgraha | 2 |
-| 3 | kṛṣṇācārya | 2 |
-| 4 | པཽཥྚཱི | 2 |
-| 5 | ānanda | 2 |
-| 6 | cakṣu | 2 |
-| 7 | link | 2 |
-| 8 | ལུམྦཱི | 2 |
-| 9 | mine | 2 |
-| 10 | vidhi | 2 |
+| 1 | སུམྦྷའི | 2 |
+| 2 | བིཀྲ | 2 |
+| 3 | jayasena | 2 |
+| 4 | ཤུདྡྷཿསརྦྦ | 2 |
+| 5 | ཧྲོཾ | 2 |
+| 6 | ཝརྞཱ | 2 |
+| 7 | caryā | 2 |
+| 8 | gīti | 2 |
+| 9 | caryāgītivṛtti | 2 |
+| 10 | དཀྲྀཏ | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 2.0020 |
-| R² (Goodness of Fit) | 0.960991 |
+| Zipf Coefficient | 2.0091 |
+| R² (Goodness of Fit) | 0.961368 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 47.5% |
-| Top 1,000 | 90.4% |
+| Top 100 | 47.6% |
+| Top 1,000 | 90.6% |
 | Top 5,000 | 99.1% |
 | Top 10,000 | 99.7% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9610 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 47.5% of corpus
-- **Long Tail:** 8,720 words needed for remaining 0.3% coverage
+- **Zipf Compliance:** R²=0.9614 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 47.6% of corpus
+- **Long Tail:** 8,977 words needed for remaining 0.3% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.8494 🏆 | 0.3707 | N/A | N/A |
-| **mono_64d** | 64 | 0.7912 | 0.3092 | N/A | N/A |
-| **mono_128d** | 128 | 0.5757 | 0.2954 | N/A | N/A |
+| **mono_32d** | 32 | 0.8542 🏆 | 0.3709 | N/A | N/A |
+| **mono_64d** | 64 | 0.8068 | 0.3078 | N/A | N/A |
+| **mono_128d** | 128 | 0.6072 | 0.2915 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8542 | 0.3660 | 0.0160 | 0.1720 |
+| **aligned_64d** | 64 | 0.8068 | 0.3152 | 0.0740 | 0.2780 |
+| **aligned_128d** | 128 | 0.6072 | 0.2869 | 0.1820 | 0.3900 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.8494 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.3251. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** mono_32d with 0.8542 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.3231. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 18.2% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **-0.603** | Low formulaic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -450,7 +485,7 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language BO appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Tibetan shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
 
 ---
 ## 7. Summary & Recommendations
@@ -461,9 +496,9 @@ The language BO appears to be more isolating or has a highly fixed vocabulary. W
 
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
-| Tokenizer | **64k BPE** | Best compression (5.30x) |
-| N-gram | **2-gram** | Lowest perplexity (469) |
-| Markov | **Context-4** | Highest predictability (89.3%) |
+| Tokenizer | **64k BPE** | Best compression (5.31x) |
+| N-gram | **2-gram** | Lowest perplexity (468) |
+| Markov | **Context-4** | Highest predictability (88.9%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
 
@@ -677,4 +712,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 07:43:59*
+*Report Date: 2026-01-03 19:39:42*

models/embeddings/aligned/bo_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:084753c7cb6163dc2d777e5b0824886aaeed6898004b2a398d7fb8123f62a58d
+size 1032429133

models/embeddings/aligned/bo_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "bo", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bo_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2eb2d57866472aeafbe80e14f631122b427d5f6bf5c57b13a10c4f2cdd0b1b22
+size 65664

models/embeddings/aligned/bo_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bo",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 741,
+  "vocab_size": 7845
+}

models/embeddings/aligned/bo_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3c170bb216c7c06c8e890facaf84b588fc4a4aec2bbd437dcbca066dbcd8222
+size 258404173

models/embeddings/aligned/bo_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "bo", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bo_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6036188735a75c7c462ee1abe6fd4c7d969876857de9d57fccfc7aa391d8797
+size 4224

models/embeddings/aligned/bo_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bo",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 741,
+  "vocab_size": 7845
+}

models/embeddings/aligned/bo_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73f9b1bd58b15af13dcf356c37edd858a42c9f3cc4c6a33cc186409c1e259077
+size 516412493

models/embeddings/aligned/bo_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "bo", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bo_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09fcdd8c7afef07dee7dd8a35e60883d0fae6f0ee0d7dbbb45a7aa3ffacf5464
+size 16512

models/embeddings/aligned/bo_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bo",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 741,
+  "vocab_size": 7845
+}

models/embeddings/monolingual/bo_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d7589a8e32a4a14dff0415d4773f3b2c21bcb132b01f0cdcdbb3fd5cac2d0a7
-size 1031854630
+oid sha256:084753c7cb6163dc2d777e5b0824886aaeed6898004b2a398d7fb8123f62a58d
+size 1032429133

models/embeddings/monolingual/bo_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 7310
+  "vocab_size": 7845
 }

models/embeddings/monolingual/bo_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:25945951b2d9bd762b4aa107b6b33d9788f48752c00df6bb74be80ee7c245721
-size 258240550
+oid sha256:f3c170bb216c7c06c8e890facaf84b588fc4a4aec2bbd437dcbca066dbcd8222
+size 258404173

models/embeddings/monolingual/bo_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 7310
+  "vocab_size": 7845
 }

models/embeddings/monolingual/bo_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:05e343618ad45867434c979594e5c58131520908c00fb32eeb884a9598ae5c2d
-size 516111910
+oid sha256:73f9b1bd58b15af13dcf356c37edd858a42c9f3cc4c6a33cc186409c1e259077
+size 516412493

models/embeddings/monolingual/bo_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 7310
+  "vocab_size": 7845
 }

models/subword_markov/bo_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9bca36abfeb85c32777f9e7088e594c535d131c8aea9a20d599c33315361efed
-size 432607
+oid sha256:c731e09b32af3cfab2f5723680f7201533f5cfa5b7fea9c31c210b22ff084ae3
+size 433406

models/subword_markov/bo_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "bo",
-  "unique_contexts": 8378,
-  "total_transitions": 22713130
+  "unique_contexts": 8393,
+  "total_transitions": 23798440
 }

models/subword_markov/bo_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f9ea9da39850551788d4eaec8b6dac589a7513f5cd8a9bc6e26566f17ac36287
-size 1811900
+oid sha256:e24401f47b18cc7f6195df6669483bf36f8e23afaff162378a9f72ea0798460d
+size 1843290

models/subword_markov/bo_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "bo",
-  "unique_contexts": 57113,
-  "total_transitions": 22700930
+  "unique_contexts": 57328,
+  "total_transitions": 23785927
 }

models/subword_markov/bo_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:592d4d34357c96997930f7a0d9aa2f06a8cef2b6ce1424d18b9c926d509600ea
-size 6648848
+oid sha256:b371d0c65f3f3c6ab8280ac56b7e1194a82adc3e06ecb550f152ebfb7a39a650
+size 6697768

models/subword_markov/bo_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "bo",
-  "unique_contexts": 233848,
-  "total_transitions": 22688730
+  "unique_contexts": 235662,
+  "total_transitions": 23773414
 }

models/subword_markov/bo_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6eb47b6d454ebb16771c614d17731ae5c2763137d58fcb511068e632490768da
-size 18752183
+oid sha256:99cbf4cee5ba3694dbe965ae72607578be5813250bdf03ead0b589f10a64d2ac
+size 19005342

models/subword_markov/bo_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "bo",
-  "unique_contexts": 765950,
-  "total_transitions": 22676530
+  "unique_contexts": 773603,
+  "total_transitions": 23760901
 }

models/subword_ngram/bo_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b741663b924d6701ddf3061df84aeb993129494f05dbf2c3f13b1b796a281ca7
-size 216011
+oid sha256:a1ac072526e1f94494222d3da83e75649088f5e0a4bd56dcb80465d251ccb34e
+size 219212

models/subword_ngram/bo_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "bo",
-  "unique_ngrams": 14734,
-  "total_ngrams": 22713130
+  "unique_ngrams": 14902,
+  "total_ngrams": 23798440
 }

models/subword_ngram/bo_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:abf34078d1e786d65a97141325854d312713c2230be0def57e7f299e03d9f735
-size 1203727
+oid sha256:7cf4b752f98d14a3e46e02f176fcc6e1276f5374b0c97b4dd76c436a930d1ab3
+size 1247296

models/subword_ngram/bo_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "bo",
-  "unique_ngrams": 86351,
-  "total_ngrams": 22700930
+  "unique_ngrams": 87521,
+  "total_ngrams": 23785927
 }

models/subword_ngram/bo_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d94ccc56cc64b3abed99bb82e501c14f9d76c46160d144d960d1f21e1afd3e74
-size 5706148
+oid sha256:30f3e3651574f6184dd55b2266ac478fde4f98d397932440db67f81a8c4d9bce
+size 5797018

models/subword_ngram/bo_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "bo",
-  "unique_ngrams": 391088,
-  "total_ngrams": 22688730
+  "unique_ngrams": 395961,
+  "total_ngrams": 23773414
 }

models/subword_ngram/bo_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f9fb0be52ec70e4e354dc2f4190c614a1d5a6d787a9d166c3d20919819729dd
+size 13535378

models/subword_ngram/bo_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "bo",
+  "unique_ngrams": 872546,
+  "total_ngrams": 23760901
+}

models/tokenizer/bo_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b80644ab76ef9f1236204c625c4e4dacd553df77f82729816a4c8648fac85d0f
-size 682438
+oid sha256:f34a7dab66b5a61db9484987f3d5a907d2a1c95b56c101a73f557593fec9d7a4
+size 682417

models/tokenizer/bo_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:099deda5fa2f785fb901ef55b737350aa4362ed293d20131e1a8d111b9cb3b59
-size 1197306
+oid sha256:5b377343285afaff84c0a99d3490f3fc80f540cb87a52bf7b48e2f66ca70bf9a
+size 1196867

models/tokenizer/bo_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e1346dc7d382bd75b96573d9f0123d6b2b1fa17c2cdfca58ee3e0289e2c5ab80
-size 2340697
+oid sha256:51d089632532888ce876861b577b00c5f13a9f45540f7b3f36c14337a2fdd7f1
+size 2341398

models/tokenizer/bo_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c734632bf663d3ae5074f8a9e3e315b68abee224608a9a41d7a6cad565ec452
-size 442308
+oid sha256:3ae671e2f3cbc76a82fab923f7c4e6994a066b20006546f3a4be7e489b2aa09a
+size 442368

models/vocabulary/bo_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cfa52cc050640e4828a3dd12a129e73d77eccd600656e7e8ecac1d73d7e04695
-size 326513
+oid sha256:7be26dfffcc7c08a8d3bbd5baab565213143190ba65d9e037ecccd9a60f2a07d
+size 330727

models/vocabulary/bo_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "bo",
-  "vocabulary_size": 18720,
+  "vocabulary_size": 18977,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.006158586066529067,
+    "type_token_ratio": 0.005952345010759796,
     "coverage": {
-      "top_100": 0.4733026861716062,
-      "top_1000": 0.9009478947369145,
-      "top_5000": 0.9874471227821341,
-      "top_10000": 0.9934588401027036
+      "top_100": 0.47476495023610643,
+      "top_1000": 0.9028692439946896,
+      "top_5000": 0.9877771234944227,
+      "top_10000": 0.9936140865251962
     },
-    "hapax_count": 26064,
-    "hapax_ratio": 0.5819935691318328,
-    "total_documents": 12200
+    "hapax_count": 26369,
+    "hapax_ratio": 0.5815066378511886,
+    "total_documents": 12513
   }
 }

models/word_markov/bo_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28be79310aca5cc300584166fdaf88159c29ab0b3c1a8a528b951d63a306c412
-size 3808759
+oid sha256:384777a1420055c01cb91824353a76ae762bc7235334bd7a5c196a699035ce66
+size 3874393

models/word_markov/bo_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "bo",
-  "unique_contexts": 44551,
-  "total_transitions": 7259599
+  "unique_contexts": 45103,
+  "total_transitions": 7605661
 }

models/word_markov/bo_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e4348c33a538330f70aeb91a4ca305d25fef537360b7e48e26baa9e5158477d4
-size 25145984
+oid sha256:28d9cf52a6e4edc120d66190763546e48dda7208747ddc8c66d03b843d4be070
+size 25624839

models/word_markov/bo_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "bo",
-  "unique_contexts": 792341,
-  "total_transitions": 7247399
+  "unique_contexts": 800524,
+  "total_transitions": 7593148
 }