Upload all models and assets for ban (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: ban
-language_name: BAN
+language_name: Balinese
 language_family: austronesian_other
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-austronesian_other
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 5.077
+    value: 5.076
   - name: best_isotropy
     type: isotropy
-    value: 0.8530
+    value: 0.8561
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# BAN - Wikilangs Models
+# Balinese - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **BAN** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Balinese** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 4.073x | 4.08 | 0.1890% | 240,149 |
-| **16k** | 4.474x | 4.48 | 0.2076% | 218,639 |
-| **32k** | 4.813x | 4.82 | 0.2234% | 203,246 |
-| **64k** | 5.077x 🏆 | 5.08 | 0.2356% | 192,667 |
+| **8k** | 4.067x | 4.07 | 0.1935% | 240,819 |
+| **16k** | 4.471x | 4.48 | 0.2127% | 219,044 |
+| **32k** | 4.812x | 4.82 | 0.2289% | 203,541 |
+| **64k** | 5.076x 🏆 | 5.08 | 0.2415% | 192,952 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Hamm (, Latin: Hammona) inggih punika kota ring Rhine-Westphalia Kalér, Jerman.`
+**Sample 1:** `920 921 922 923 924 925 926 927 928 929 Jadma Embas Seda Pustaka Pranala liyané ...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁ham m ▁(, ▁latin : ▁ham m ona ) ▁inggih ... (+12 more)` | 22 |
-| 16k | `▁ham m ▁(, ▁latin : ▁ham m ona ) ▁inggih ... (+10 more)` | 20 |
-| 32k | `▁ham m ▁(, ▁latin : ▁ham m ona ) ▁inggih ... (+10 more)` | 20 |
-| 64k | `▁hamm ▁(, ▁latin : ▁hamm ona ) ▁inggih ▁punika ▁kota ... (+8 more)` | 18 |
+| 8k | `▁ 9 2 0 ▁ 9 2 1 ▁ 9 ... (+40 more)` | 50 |
+| 16k | `▁ 9 2 0 ▁ 9 2 1 ▁ 9 ... (+40 more)` | 50 |
+| 32k | `▁ 9 2 0 ▁ 9 2 1 ▁ 9 ... (+40 more)` | 50 |
+| 64k | `▁ 9 2 0 ▁ 9 2 1 ▁ 9 ... (+40 more)` | 50 |
 
-**Sample 2:** `Kharkiv (), utawi Kharkov () inggih punika kota pinih ageng kakalih ring Ukraina...`
+**Sample 2:** `Reutlingen (; Swabia: Reitlenga) inggih punika sinunggil kota ring Baden-Württem...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁kh ark iv ▁(), ▁utawi ▁kh ark ov ▁() ▁inggih ... (+24 more)` | 34 |
-| 16k | `▁kh ark iv ▁(), ▁utawi ▁kh ark ov ▁() ▁inggih ... (+22 more)` | 32 |
-| 32k | `▁kh ark iv ▁(), ▁utawi ▁kh ark ov ▁() ▁inggih ... (+22 more)` | 32 |
-| 64k | `▁kharkiv ▁(), ▁utawi ▁khark ov ▁() ▁inggih ▁punika ▁kota ▁pinih ... (+15 more)` | 25 |
+| 8k | `▁re ut ling en ▁(; ▁sw ab ia : ▁re ... (+34 more)` | 44 |
+| 16k | `▁re ut ling en ▁(; ▁sw ab ia : ▁re ... (+28 more)` | 38 |
+| 32k | `▁re ut lingen ▁(; ▁sw abia : ▁re it l ... (+25 more)` | 35 |
+| 64k | `▁reut lingen ▁(; ▁sw abia : ▁re it l enga ... (+22 more)` | 32 |
 
-**Sample 3:** `Brasília (;"Brasilia" (US) tur ) inggih punika ibu kota saking Brasil. Pustaka`
+**Sample 3:** `Terneuzen () inggih punika kota miwah kotamadya ring sisi kelod kauh Belanda, ri...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁br as í l ia ▁(; " br asil ia ... (+14 more)` | 24 |
-| 16k | `▁br as í lia ▁(; " br asil ia " ... (+13 more)` | 23 |
-| 32k | `▁brasília ▁(; " br asil ia " ▁( us ) ... (+10 more)` | 20 |
-| 64k | `▁brasília ▁(;" brasil ia " ▁( us ) ▁tur ▁) ... (+8 more)` | 18 |
+| 8k | `▁ter ne uz en ▁() ▁inggih ▁punika ▁kota ▁miwah ▁kotamadya ... (+21 more)` | 31 |
+| 16k | `▁ter ne uz en ▁() ▁inggih ▁punika ▁kota ▁miwah ▁kotamadya ... (+17 more)` | 27 |
+| 32k | `▁ter ne uz en ▁() ▁inggih ▁punika ▁kota ▁miwah ▁kotamadya ... (+15 more)` | 25 |
+| 64k | `▁ter ne uzen ▁() ▁inggih ▁punika ▁kota ▁miwah ▁kotamadya ▁ring ... (+14 more)` | 24 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 5.077x compression
-- **Lowest UNK Rate:** 8k with 0.1890% unknown tokens
+- **Best Compression:** 64k achieves 5.076x compression
+- **Lowest UNK Rate:** 8k with 0.1935% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 4,798 | 12.23 | 59,688 | 35.6% | 57.3% |
-| **2-gram** | Subword | 225 🏆 | 7.81 | 7,788 | 73.4% | 99.2% |
-| **3-gram** | Word | 5,769 | 12.49 | 77,113 | 33.5% | 55.7% |
-| **3-gram** | Subword | 1,669 | 10.70 | 42,522 | 31.2% | 79.1% |
-| **4-gram** | Word | 8,680 | 13.08 | 116,715 | 28.6% | 51.0% |
-| **4-gram** | Subword | 7,684 | 12.91 | 208,144 | 18.1% | 53.6% |
+| **2-gram** | Word | 4,640 | 12.18 | 61,259 | 36.3% | 57.8% |
+| **2-gram** | Subword | 223 🏆 | 7.80 | 8,004 | 73.6% | 99.2% |
+| **3-gram** | Word | 5,627 | 12.46 | 79,401 | 34.2% | 56.0% |
+| **3-gram** | Subword | 1,643 | 10.68 | 43,230 | 31.4% | 79.4% |
+| **4-gram** | Word | 8,547 | 13.06 | 120,311 | 29.1% | 51.2% |
+| **4-gram** | Subword | 7,491 | 12.87 | 210,661 | 18.4% | 54.1% |
+| **5-gram** | Word | 8,777 | 13.10 | 92,971 | 25.7% | 49.1% |
+| **5-gram** | Subword | 21,126 | 14.37 | 563,270 | 15.0% | 42.7% |
 
 ### Top 5 N-grams by Size
 
@@ -150,68 +162,88 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `situs resmi` | 41,099 |
-| 2 | `inggih punika` | 37,495 |
-| 3 | `silih tunggil` | 22,082 |
-| 4 | `pranala jaba` | 21,960 |
-| 5 | `pusat statistik` | 21,725 |
+| 1 | `situs resmi` | 43,663 |
+| 2 | `inggih punika` | 39,149 |
+| 3 | `pusat statistik` | 24,769 |
+| 4 | `badan pusat` | 24,755 |
+| 5 | `silih tunggil` | 23,231 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `badan pusat statistik` | 21,708 |
-| 2 | `pustaka pranala jaba` | 20,507 |
-| 3 | `inggih punika silih` | 19,377 |
-| 4 | `punika silih tunggil` | 19,020 |
-| 5 | `pranala jaba situs` | 17,860 |
+| 1 | `badan pusat statistik` | 24,753 |
+| 2 | `pustaka pranala jaba` | 21,680 |
+| 3 | `inggih punika silih` | 20,522 |
+| 4 | `punika silih tunggil` | 20,156 |
+| 5 | `pranala jaba situs` | 19,252 |
 
 **4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `inggih punika silih tunggil` | 18,913 |
-| 2 | `pranala jaba situs resmi` | 17,672 |
-| 3 | `pustaka pranala jaba situs` | 17,290 |
-| 4 | `dados kauahin ilang yening` | 14,166 |
-| 5 | `kauahin ilang yening url` | 13,881 |
+| 1 | `inggih punika silih tunggil` | 20,046 |
+| 2 | `pranala jaba situs resmi` | 19,034 |
+| 3 | `pustaka pranala jaba situs` | 18,664 |
+| 4 | `dados kauahin ilang yening` | 15,610 |
+| 5 | `kauahin ilang yening url` | 15,325 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `pustaka pranala jaba situs resmi` | 18,475 |
+| 2 | `dados kauahin ilang yening url` | 15,325 |
+| 3 | `kauahin ilang yening url nenten` | 15,194 |
+| 4 | `url dados kauahin ilang yening` | 15,039 |
+| 5 | `ilang yening url nenten aktip` | 14,998 |
 
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `a n` | 880,577 |
-| 2 | `n g` | 735,053 |
-| 3 | `a _` | 536,413 |
-| 4 | `i n` | 523,219 |
-| 5 | `n _` | 516,092 |
+| 1 | `a n` | 914,478 |
+| 2 | `n g` | 765,351 |
+| 3 | `a _` | 556,979 |
+| 4 | `i n` | 546,378 |
+| 5 | `n _` | 539,027 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `n g _` | 361,156 |
-| 2 | `a n _` | 287,413 |
-| 3 | `i n g` | 287,067 |
-| 4 | `a n g` | 219,608 |
-| 5 | `_ k a` | 213,760 |
+| 1 | `n g _` | 376,926 |
+| 2 | `a n _` | 301,627 |
+| 3 | `i n g` | 300,756 |
+| 4 | `a n g` | 227,744 |
+| 5 | `_ k a` | 223,144 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `i n g _` | 219,518 |
-| 2 | `r i n g` | 145,165 |
-| 3 | `_ r i n` | 128,090 |
-| 4 | `a n g _` | 86,655 |
-| 5 | `u n i k` | 72,566 |
+| 1 | `i n g _` | 230,681 |
+| 2 | `r i n g` | 152,062 |
+| 3 | `_ r i n` | 133,355 |
+| 4 | `a n g _` | 89,274 |
+| 5 | `u n i k` | 75,300 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `r i n g _` | 149,014 |
+| 2 | `_ r i n g` | 133,072 |
+| 3 | `p u n i k` | 74,857 |
+| 4 | `_ p u n i` | 72,286 |
+| 5 | `b u p a t` | 70,377 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram (subword) with 225
+- **Best Perplexity:** 2-gram (subword) with 223
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~54% of corpus
+- **Coverage:** Top-1000 patterns cover ~43% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.7212 | 1.649 | 5.13 | 253,714 | 27.9% |
-| **1** | Subword | 0.9714 | 1.961 | 7.03 | 4,633 | 2.9% |
-| **2** | Word | 0.2297 | 1.173 | 1.53 | 1,298,868 | 77.0% |
-| **2** | Subword | 0.6107 | 1.527 | 3.55 | 32,560 | 38.9% |
-| **3** | Word | 0.0749 | 1.053 | 1.14 | 1,983,308 | 92.5% |
-| **3** | Subword | 0.5954 | 1.511 | 3.32 | 115,474 | 40.5% |
-| **4** | Word | 0.0289 🏆 | 1.020 | 1.05 | 2,240,261 | 97.1% |
-| **4** | Subword | 0.6610 | 1.581 | 2.96 | 383,801 | 33.9% |
+| **1** | Word | 0.7231 | 1.651 | 5.15 | 258,667 | 27.7% |
+| **1** | Subword | 0.9698 | 1.959 | 7.06 | 4,719 | 3.0% |
+| **2** | Word | 0.2300 | 1.173 | 1.54 | 1,327,861 | 77.0% |
+| **2** | Subword | 0.6130 | 1.529 | 3.55 | 33,296 | 38.7% |
+| **3** | Word | 0.0751 | 1.053 | 1.14 | 2,029,547 | 92.5% |
+| **3** | Subword | 0.5903 | 1.506 | 3.30 | 118,157 | 41.0% |
+| **4** | Word | 0.0289 🏆 | 1.020 | 1.05 | 2,293,918 | 97.1% |
+| **4** | Subword | 0.6581 | 1.578 | 2.95 | 389,827 | 34.2% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `ring warsa puniki dados kauahin ilang yening url dados kaapus saking sistem ekologi dan bedah langsu...`
-2. `kabupatén kediri jawa timur pustaka pranala jaba situs resmi pamréntahan wali ngancan ngamokohang ba...`
-3. `punika silih tunggil désa ring thailand punika wenten ring sérial mabasis ring wewidangan kecamatan ...`
+1. `ring kabupatén manggarai univérsitas téknologi langkungan saking lis kediri propinsi jawa timur situ...`
+2. `kabupatén bandar udara sipil negara wagian connecticut john musker dave akbarshah fikarno partai pol...`
+3. `punika silih tunggil gampong ring panguntat warsa perang sane madaging aglomerasi pays blanc kawentu...`
 
 **Context Size 2:**
 
-1. `situs resmi pamréntahan kabupatén bima badan pusat statistik kota bengkulu badan pusat statistik pro...`
-2. `inggih punika silih sinunggil gendingan tradisional thailand sane pinih sering kacingak pinaka gerha...`
-3. `silih tunggil pagending tur ngamedalang surat kaputusan nomor sadurugnyane ring warsa akéh kramanyan...`
+1. `situs resmi provinsi kalimantan timur indonésia pustaka pranala jaba of the betawi and their subordi...`
+2. `inggih punika silih tunggil désa dinas sané magenah ring désa karimunjawa pulau karimunjawa gua sara...`
+3. `pusat statistik provinsi lampung badan pusat statistik nusa tenggara timur ring panegara indonésia p...`
 
 **Context Size 3:**
 
-1. `badan pusat statistik propinsi jawa tengah indonésia mawit saking pérméndagri nomor 137 warsa indik ...`
-2. `pustaka pranala jaba situs resmi propinsi bali badan pusat statistik propinsi kalimantan selatan bad...`
-3. `inggih punika silih tunggil kecamatan ring kabupatén timor tengah utara ring nusa tenggara timur bad...`
+1. `badan pusat statistik provinsi lampung badan pusat statistik provinsi banten situs resmi pemerintah ...`
+2. `pustaka pranala jaba situs resmi pamréntahan kota malang prodeskel binapemdes kemendagri banyuwangi ...`
+3. `inggih punika silih tunggil kecamatan ring kabupatén tuban ring jawa timur ring panegara indonésia p...`
 
 **Context Size 4:**
 
-1. `inggih punika silih tunggil désa ring kecamatan pulau pulau kur tual propinsi maluku indonésia pusta...`
-2. `pranala jaba situs resmi pamrentahan provinsi kepulauan bangka belitung badan pusat statistik kabupa...`
-3. `pustaka pranala jaba situs resmi pamrentahan provinsi kepulauan bangka belitung badan pusat statisti...`
+1. `inggih punika silih tunggil désa dinas sané magenah ring kecamatan pakem ring wawengkon kabupatén bo...`
+2. `pranala jaba situs resmi pamréntahan propinsi kalimantan tengah badan pusat statistik propinsi kalim...`
+3. `pustaka pranala jaba situs resmi pamrentahan propinsi jawa tengah badan pusat statistik propinsi daé...`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `a._gané_l_i,_dan`
-2. `_ptrandopi_mi_ba`
-3. `n_107_sika,_dika`
+1. `akraning_pa_dang`
+2. `_pawewen,_ako_in`
+3. `ngkang_l_parasih`
 
 **Context Size 2:**
 
-1. `angaing_wawewidué`
-2. `ng_gu_kin_éman_no`
-3. `a_]_garingang_lat`
+1. `an_punisi_ka_ma_i`
+2. `ng_doh_for,_namas`
+3. `a_matasur_sur_jaj`
 
 **Context Size 3:**
 
-1. `ng_kabupatén_sané_`
-2. `an_kaapustaka_miwa`
-3. `inggih_tunggih_pas`
+1. `ng_pamréntahan_kaa`
+2. `an_sumelaya,_propi`
+3. `ing_richoir,_jani_`
 
 **Context Size 4:**
 
-1. `ing_basa_badan_pran`
-2. `ring_kaapus_sané_ri`
-3. `_ring_kabupatén_kah`
+1. `ing_lis._gresik_pun`
+2. `ring_radeship_himse`
+3. `_ring_soroh_jaya_be`
 
 
 ### Key Findings
 
 - **Best Predictability:** Context-4 (word) with 97.1% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (383,801 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (389,827 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,64 +346,64 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 96,177 |
-| Total Tokens | 3,540,495 |
-| Mean Frequency | 36.81 |
+| Vocabulary Size | 98,403 |
+| Total Tokens | 3,677,636 |
+| Mean Frequency | 37.37 |
 | Median Frequency | 3 |
-| Frequency Std Dev | 739.04 |
+| Frequency Std Dev | 767.63 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | ring | 127,899 |
-| 2 | kabupatén | 58,514 |
-| 3 | punika | 50,657 |
-| 4 | sané | 45,835 |
-| 5 | situs | 44,988 |
-| 6 | resmi | 42,224 |
-| 7 | inggih | 37,927 |
-| 8 | saking | 37,341 |
-| 9 | url | 32,061 |
-| 10 | miwah | 31,507 |
+| 1 | ring | 133,161 |
+| 2 | kabupatén | 61,962 |
+| 3 | punika | 52,592 |
+| 4 | situs | 47,934 |
+| 5 | sané | 47,011 |
+| 6 | resmi | 44,807 |
+| 7 | inggih | 39,587 |
+| 8 | saking | 39,350 |
+| 9 | url | 35,045 |
+| 10 | propinsi | 33,485 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | kitou | 2 |
-| 2 | sialet | 2 |
-| 3 | dibanda | 2 |
-| 4 | ᬦᬶᬲᬫ᭄ | 2 |
-| 5 | reuba | 2 |
-| 6 | reuleut | 2 |
-| 7 | rheue | 2 |
-| 8 | uleue | 2 |
-| 9 | muling | 2 |
-| 10 | sanderling | 2 |
+| 1 | ᬧᬳᬗᬿ | 2 |
+| 2 | ᬧᬓᬓ᭄ | 2 |
+| 3 | ᬮᬸᬦᬸᬓ᭄ | 2 |
+| 4 | ᬫᭂᬭᬜ᭄ᬘᬂ | 2 |
+| 5 | patonangi | 2 |
+| 6 | ᬩᬩᬭᬶᬲ᭄ | 2 |
+| 7 | ᬢᬢᬓᬦ᭄ | 2 |
+| 8 | ᬳᬮᬢ᭄ | 2 |
+| 9 | ᬩᬤᭁᬦ᭄ | 2 |
+| 10 | ᬩᬮᬸᬓᬸᬂ | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 1.1306 |
-| R² (Goodness of Fit) | 0.997983 |
+| Zipf Coefficient | 1.1326 |
+| R² (Goodness of Fit) | 0.997911 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 44.6% |
-| Top 1,000 | 68.9% |
-| Top 5,000 | 82.9% |
-| Top 10,000 | 87.9% |
+| Top 100 | 45.3% |
+| Top 1,000 | 69.2% |
+| Top 5,000 | 83.1% |
+| Top 10,000 | 88.0% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9980 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 44.6% of corpus
-- **Long Tail:** 86,177 words needed for remaining 12.1% coverage
+- **Zipf Compliance:** R²=0.9979 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 45.3% of corpus
+- **Long Tail:** 88,403 words needed for remaining 12.0% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.8530 🏆 | 0.3516 | N/A | N/A |
-| **mono_64d** | 64 | 0.8495 | 0.2832 | N/A | N/A |
-| **mono_128d** | 128 | 0.8092 | 0.2232 | N/A | N/A |
+| **mono_32d** | 32 | 0.8561 🏆 | 0.3559 | N/A | N/A |
+| **mono_64d** | 64 | 0.8453 | 0.2824 | N/A | N/A |
+| **mono_128d** | 128 | 0.8108 | 0.2152 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8561 | 0.3499 | 0.0500 | 0.3000 |
+| **aligned_64d** | 64 | 0.8453 | 0.2791 | 0.1160 | 0.4180 |
+| **aligned_128d** | 128 | 0.8108 | 0.2217 | 0.1860 | 0.5760 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.8530 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.2860. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** mono_32d with 0.8561 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.2840. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 18.6% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **0.148** | Low formulaic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -426,19 +461,17 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Prefixes
 | Prefix | Examples |
 |--------|----------|
-| `-ma` | martins, masduki, maffin |
-| `-ka` | kaméloh, kaaranin, kasum |
-| `-pa` | palopat, panandatanganan, pail |
-| `-pe` | peting, pencok, pemantauan |
+| `-ka` | kaumahné, kambilo, karangdinoyo |
+| `-ma` | maseosan, matogu, manufaktur |
+| `-pa` | papadun, palmerah, pacing |
 
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-n` | baharuddin, setyawan, roussillon |
-| `-an` | setyawan, panandatanganan, mengupayakan |
-| `-ng` | peting, speaking, sanderling |
-| `-ang` | tenggarang, lendang, nguwahang |
-| `-né` | leluhurnyané, putranidané, bébékné |
+| `-n` | alien, gejeran, hughenden |
+| `-an` | gejeran, maseosan, matangnyan |
+| `-ng` | wyoming, siung, yèning |
+| `-ang` | nelebang, renang, hilirundang |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -446,18 +479,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `anga` | 1.47x | 361 contexts | nanga, sanga, hanga |
-| `ngan` | 1.54x | 182 contexts | angan, ingan, tengan |
-| `nten` | 1.71x | 86 contexts | inten, enten, wnten |
-| `atan` | 1.52x | 149 contexts | vatan, gatan, matan |
-| `ungg` | 1.55x | 117 contexts | tungg, ungga, unggun |
-| `akin` | 1.88x | 41 contexts | aking, yakin, dakin |
-| `nggi` | 1.58x | 73 contexts | anggi, nggih, senggi |
-| `taha` | 1.90x | 32 contexts | tahai, tahap, tahan |
-| `ggih` | 2.03x | 22 contexts | nggih, lnggih, inggih |
-| `ados` | 2.01x | 22 contexts | dados, sados, padosa |
-| `isti` | 1.61x | 36 contexts | bistik, sistim, pistia |
-| `cama` | 1.87x | 19 contexts | camat, camas, camah |
+| `anga` | 1.63x | 366 contexts | angar, ranga, manga |
+| `nten` | 1.91x | 86 contexts | inten, enten, wnten |
+| `atan` | 1.68x | 151 contexts | batan, vatan, patan |
+| `ngan` | 1.50x | 185 contexts | ingan, angan, ringan |
+| `akin` | 1.95x | 42 contexts | makin, dakin, yakin |
+| `ungg` | 1.47x | 120 contexts | tungg, ungga, unggak |
+| `nggi` | 1.58x | 77 contexts | anggi, nggih, ninggi |
+| `taha` | 1.86x | 33 contexts | tahan, tahai, tahar |
+| `ados` | 2.09x | 21 contexts | dados, sados, padosa |
+| `ggih` | 1.99x | 22 contexts | nggih, inggih, lnggih |
+| `stat` | 1.88x | 20 contexts | state, stats, istat |
+| `isti` | 1.56x | 37 contexts | sistim, bistik, mistik |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -465,16 +498,16 @@ This table shows which prefixes and suffixes most frequently co-occur on the sam
 
 | Prefix | Suffix | Frequency | Examples |
 |--------|--------|-----------|----------|
-| `-pa` | `-n` | 112 words | palimunan, pawacanan |
-| `-ka` | `-n` | 112 words | kamerdékaan, kagenahin |
-| `-pa` | `-an` | 96 words | palimunan, pawacanan |
-| `-pe` | `-n` | 92 words | perhubungan, penyaringan |
-| `-pe` | `-an` | 81 words | perhubungan, penyaringan |
-| `-ka` | `-ng` | 77 words | kaidipang, kawedharang |
-| `-ka` | `-ang` | 61 words | kaidipang, kawedharang |
-| `-ka` | `-an` | 56 words | kamerdékaan, kamaharajan |
-| `-ma` | `-n` | 55 words | marepan, mapitungan |
-| `-ma` | `-an` | 39 words | marepan, mapitungan |
+| `-ka` | `-n` | 119 words | kapribadian, kaanyarin |
+| `-pa` | `-n` | 117 words | palimanan, pawedaran |
+| `-pa` | `-an` | 104 words | palimanan, pawedaran |
+| `-ka` | `-ng` | 90 words | kagampilang, kalaliang |
+| `-ka` | `-ang` | 75 words | kagampilang, kalaliang |
+| `-ka` | `-an` | 68 words | kapribadian, kalanguan |
+| `-ma` | `-n` | 45 words | malun, maroon |
+| `-ma` | `-an` | 36 words | madénan, mabinaan |
+| `-ma` | `-ng` | 34 words | mamantang, mahondang |
+| `-ma` | `-ang` | 20 words | mamantang, mahondang |
 
 ### 6.5 Recursive Morpheme Segmentation
 
@@ -482,26 +515,26 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
-| kauningan | **`ka-uning-an`** | 6.0 | `uning` |
-| kaorganisasiang | **`ka-organisasi-ang`** | 6.0 | `organisasi` |
-| kakaonang | **`ka-ka-onang`** | 6.0 | `onang` |
-| pasilihan | **`pa-silih-an`** | 6.0 | `silih` |
-| kajahatan | **`ka-jahat-an`** | 6.0 | `jahat` |
-| kasedukan | **`ka-seduk-an`** | 6.0 | `seduk` |
-| kalaporang | **`ka-lapor-ang`** | 6.0 | `lapor` |
-| kakuasaan | **`ka-kuasa-an`** | 6.0 | `kuasa` |
-| padruwénan | **`pa-druwén-an`** | 6.0 | `druwén` |
-| palekadan | **`pa-lekad-an`** | 6.0 | `lekad` |
-| mategakan | **`ma-tegak-an`** | 6.0 | `tegak` |
-| kaungkabang | **`ka-ungkab-ang`** | 6.0 | `ungkab` |
-| kauwugang | **`ka-uwug-ang`** | 6.0 | `uwug` |
-| panyambung | **`pa-nyambu-ng`** | 6.0 | `nyambu` |
-| panularan | **`pa-nular-an`** | 6.0 | `nular` |
+| patarungan | **`pa-taru-ng-an`** | 7.5 | `taru` |
+| kalédangan | **`ka-léda-ng-an`** | 7.5 | `léda` |
+| malimongan | **`ma-limo-ng-an`** | 7.5 | `limo` |
+| kasemaran | **`ka-semar-an`** | 6.0 | `semar` |
+| kaasosiasiang | **`ka-asosiasi-ang`** | 6.0 | `asosiasi` |
+| kadaftarang | **`ka-daftar-ang`** | 6.0 | `daftar` |
+| malaibang | **`ma-laib-ang`** | 6.0 | `laib` |
+| kasunanan | **`ka-sunan-an`** | 6.0 | `sunan` |
+| kawarisang | **`ka-waris-ang`** | 6.0 | `waris` |
+| pangabdian | **`pa-ngabdi-an`** | 6.0 | `ngabdi` |
+| palaibang | **`pa-laib-ang`** | 6.0 | `laib` |
+| kabudayaan | **`ka-budaya-an`** | 6.0 | `budaya` |
+| mapangangge | **`ma-pa-ngangge`** | 6.0 | `ngangge` |
+| mapontang | **`ma-pont-ang`** | 6.0 | `pont` |
+| kajegegan | **`ka-jegeg-an`** | 6.0 | `jegeg` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language BAN appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Balinese shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
 
 ---
 ## 7. Summary & Recommendations
@@ -513,7 +546,7 @@ The language BAN appears to be more isolating or has a highly fixed vocabulary.
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
 | Tokenizer | **64k BPE** | Best compression (5.08x) |
-| N-gram | **2-gram** | Lowest perplexity (225) |
+| N-gram | **2-gram** | Lowest perplexity (223) |
 | Markov | **Context-4** | Highest predictability (97.1%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
@@ -728,4 +761,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 06:12:56*
+*Report Date: 2026-01-03 18:39:33*

models/embeddings/aligned/ban_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa7015dba1b32c26d14135f707769a38af9822fb21669e7e91ac54d8bb06d204
+size 1070163526

models/embeddings/aligned/ban_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "ban", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/ban_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bb3bf0c6d4d0d945a8bfec3bc7023135a84d2c295c5ec74f3b31430a0885d79
+size 65664

models/embeddings/aligned/ban_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "ban",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 14715,
+  "vocab_size": 44349
+}

models/embeddings/aligned/ban_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63e94f2703c400ae51ea4776cafae9113ef451be47680e796e6b68a8b15e7c62
+size 268103494

models/embeddings/aligned/ban_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "ban", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/ban_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4609b2ed15b00f8a3811474e5c9d953a1d951b9e23a7a6be7c50a6742e4c8ca8
+size 4224

models/embeddings/aligned/ban_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "ban",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 14715,
+  "vocab_size": 44349
+}

models/embeddings/aligned/ban_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0723c484eafe35591778bd74c72acce587929b986611dbef45b5f40e3625f00f
+size 535456838

models/embeddings/aligned/ban_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "ban", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/ban_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09a4bb28588585c56283e814869b4c1ddee25abce34e1e5aad4b39ed3d864aa3
+size 16512

models/embeddings/aligned/ban_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "ban",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 14715,
+  "vocab_size": 44349
+}

models/embeddings/monolingual/ban_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bdf8bd41f143ec229b2b83e523def59ca3894f5e32c88dc9ad1d5777bb3cdbc1
-size 1069224476
+oid sha256:fa7015dba1b32c26d14135f707769a38af9822fb21669e7e91ac54d8bb06d204
+size 1070163526

models/embeddings/monolingual/ban_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 43447
+  "vocab_size": 44349
 }

models/embeddings/monolingual/ban_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c7b5b04af24d58eb2352605fd5a6a3884aa17bd5211923840f1d2399995b3dc
-size 267857180
+oid sha256:63e94f2703c400ae51ea4776cafae9113ef451be47680e796e6b68a8b15e7c62
+size 268103494

models/embeddings/monolingual/ban_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 43447
+  "vocab_size": 44349
 }

models/embeddings/monolingual/ban_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45be3ee34bd2a796319b97a63a3801529c7d2b0804d891a90aa7697ac777fcda
-size 534979612
+oid sha256:0723c484eafe35591778bd74c72acce587929b986611dbef45b5f40e3625f00f
+size 535456838

models/embeddings/monolingual/ban_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 43447
+  "vocab_size": 44349
 }

models/subword_markov/ban_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b557a43cbf89ec6e5b8fe521107f375a803257117295fa8f2f26d54017e944ef
-size 238064
+oid sha256:718eb39c64f6851b45cb95f99bbf810ec96f2cc076e64f33264f4d1cf13335c0
+size 242588

models/subword_markov/ban_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "ban",
-  "unique_contexts": 4633,
-  "total_transitions": 26072638
+  "unique_contexts": 4719,
+  "total_transitions": 27076397
 }

models/subword_markov/ban_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc162e729fc9f1b78bdc516d217698eb4c1e8af546cf0d4b64801623d1296b5e
-size 1052622
+oid sha256:c3e9a8e2a0e7dde391cd98d4038c21d582c6dbf7a9646e0b5464a19d6ed97b0a
+size 1066241

models/subword_markov/ban_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "ban",
-  "unique_contexts": 32560,
-  "total_transitions": 26039719
+  "unique_contexts": 33296,
+  "total_transitions": 27042034
 }

models/subword_markov/ban_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:62c18ce24bb55f6d1333346623229b9025cc0f78b411f131da48dbb5f6b5bfaf
-size 3150588
+oid sha256:1ecac46deaa421d98934c44e71c7594b179c4b50ff14dcea186802238def94c0
+size 3262707

models/subword_markov/ban_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "ban",
-  "unique_contexts": 115474,
-  "total_transitions": 26006800
+  "unique_contexts": 118157,
+  "total_transitions": 27007671
 }

models/subword_markov/ban_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:beb82bbbccc1e498763133c2ca43a33d34399701a3feadc33b33d7e28db10c2d
-size 9163427
+oid sha256:a596bdbeb821323250ec0d52fb8ba1f6ad34bb3ea2159a62adf907a7b29ed02a
+size 9319931

models/subword_markov/ban_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "ban",
-  "unique_contexts": 383801,
-  "total_transitions": 25973881
+  "unique_contexts": 389827,
+  "total_transitions": 26973308
 }

models/subword_ngram/ban_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8d06b7f0eea6caff12eb48816bcfb2e270b3b2491f26727e3f0514ff35fc746
-size 106802
+oid sha256:7fcbc05f1272c5b1332f43cff8a211618970272f4a7a82ab81d4dcd7b6c5e1e8
+size 109826

models/subword_ngram/ban_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "ban",
-  "unique_ngrams": 7788,
-  "total_ngrams": 26072638
+  "unique_ngrams": 8004,
+  "total_ngrams": 27076397
 }

models/subword_ngram/ban_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9343f458518ed0d1f0a407a342e9b5cecbee42b529784e0dcb0b72918b79d1d3
-size 564805
+oid sha256:eb4d89ef3e14a34a16aa3d600715a522ff2d826f7146a7c998bbe3d7c1c67c9f
+size 584367

models/subword_ngram/ban_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "ban",
-  "unique_ngrams": 42522,
-  "total_ngrams": 26039719
+  "unique_ngrams": 43230,
+  "total_ngrams": 27042034
 }

models/subword_ngram/ban_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b5c1474de28fe82c885eb32b731cad7f32987a3ef6b82a9ab47b3b48c9e2ce1a
-size 2348974
+oid sha256:6b15b80930e90d5cbcb46b4cf6f091b611cfcfa749c7466d6db49385f7012264
+size 2381813

models/subword_ngram/ban_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "ban",
-  "unique_ngrams": 208144,
-  "total_ngrams": 26006800
+  "unique_ngrams": 210661,
+  "total_ngrams": 27007671
 }

models/subword_ngram/ban_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df02a3767b920d1e2f47e25adb9c269988bbba584f47676d81c99c80df21c3fb
+size 6441439

models/subword_ngram/ban_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "ban",
+  "unique_ngrams": 563270,
+  "total_ngrams": 26973308
+}

models/tokenizer/ban_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0194959437d4a8b04b5d3f37d4b95caf926ecb7c4136e157e06f185417710348
-size 507366
+oid sha256:88b097bec96c86be17dda7b0ac3181adb2c854d9d5411be044f3b3f6b70daeb1
+size 507335

models/tokenizer/ban_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d9e73114a8bfb09cc05cf62d77156539332a4b993d735d59704b21d2055bd5b
-size 785247
+oid sha256:a46272b91e6d11d1d893a911a66a497ebce18277c2e91cf14bc371de57238bea
+size 784420

models/tokenizer/ban_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c4163cfc92fedee34979f21e46b3960651c5a1aea63e481f06e935586c4982c
-size 1355653
+oid sha256:3b04a538e67cae4d48d961ecdd8a0a02657476f42538b1a28741d9cf63ad0c1c
+size 1360288

models/tokenizer/ban_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79dc41677c1fda74a9243a5e8e6b256d3a8d6e89e202f72fe68235e2c50ce0df
-size 372914
+oid sha256:c6f6acca047a065b29e4e6406c18bd7d49a7637ab851303ea7a466119f4f94a4
+size 372967

models/vocabulary/ban_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e4e8f2a999ef9ad888861be852297181f3f88c4f902b2c2d3a001b3f88a5eeb
-size 1669252
+oid sha256:8bfcfa84a48dadb625735b3a0c274277d625a336e6309d08a4092de0a5f31862
+size 1701426

models/vocabulary/ban_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "ban",
-  "vocabulary_size": 96177,
+  "vocabulary_size": 98403,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.06865216883420294,
+    "type_token_ratio": 0.06744349463678129,
     "coverage": {
-      "top_100": 0.4271482296290528,
-      "top_1000": 0.6597757616661908,
-      "top_5000": 0.7939794084053682,
-      "top_10000": 0.8415540715935934
+      "top_100": 0.43393023659963936,
+      "top_1000": 0.6627173901998574,
+      "top_5000": 0.7959204676704827,
+      "top_10000": 0.8428477520181278
     },
-    "hapax_count": 157713,
-    "hapax_ratio": 0.6211863405411793,
-    "total_documents": 32919
+    "hapax_count": 160451,
+    "hapax_ratio": 0.6198513447735017,
+    "total_documents": 34363
   }
 }

models/word_markov/ban_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b32e21376297d35ef6bbd5d4573db01b79bce03f68060112590a5eccf22d3f2
-size 12122512
+oid sha256:58bd88c2ff7b6fedad91fc291fddfc0eb362aff36cbd25ce91ac0f5e68afd618
+size 12427971

models/word_markov/ban_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "ban",
-  "unique_contexts": 253714,
-  "total_transitions": 3665289
+  "unique_contexts": 258667,
+  "total_transitions": 3803724
 }

models/word_markov/ban_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0cc40598890a3afe8525cbf4abca4b849a1276159d24aed089a152527e3bc5a
-size 26358283
+oid sha256:aaef8ba53b9b5f49bb321f3ae6c3e7f0a609ebeb34d6e66e1bfed960a4837c3a
+size 27002495

models/word_markov/ban_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "ban",
-  "unique_contexts": 1298868,
-  "total_transitions": 3632370
+  "unique_contexts": 1327861,
+  "total_transitions": 3769361
 }