Upload all models and assets for btm (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: btm
-language_name: BTM
+language_name: Batak Mandailing
 language_family: austronesian_batak
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-austronesian_batak
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -26,17 +36,17 @@ metrics:
     value: 5.210
   - name: best_isotropy
     type: isotropy
-    value: 0.3926
+    value: 0.4518
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# BTM - Wikilangs Models
+# Batak Mandailing - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **BTM** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Batak Mandailing** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 4.162x | 4.17 | 0.0869% | 217,411 |
-| **16k** | 4.607x | 4.61 | 0.0962% | 196,367 |
-| **32k** | 5.005x | 5.01 | 0.1045% | 180,776 |
-| **64k** | 5.210x 🏆 | 5.22 | 0.1088% | 173,672 |
+| **8k** | 4.164x | 4.17 | 0.0881% | 216,736 |
+| **16k** | 4.609x | 4.61 | 0.0975% | 195,810 |
+| **32k** | 5.005x | 5.01 | 0.1059% | 180,321 |
+| **64k** | 5.210x 🏆 | 5.22 | 0.1103% | 173,224 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Natal ima sada kecamatan di Kabupaten Mandailing Natal, Sumatera Utara, Indonesi...`
+**Sample 1:** `Kumpulan Setia ima sala sada huta na adong i kecamatan Huta Bargot, kabupaten Ma...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁natal ▁ima ▁sada ▁kecamatan ▁di ▁kabupaten ▁mandailing ▁natal , ▁sumatera ... (+4 more)` | 14 |
-| 16k | `▁natal ▁ima ▁sada ▁kecamatan ▁di ▁kabupaten ▁mandailing ▁natal , ▁sumatera ... (+4 more)` | 14 |
-| 32k | `▁natal ▁ima ▁sada ▁kecamatan ▁di ▁kabupaten ▁mandailing ▁natal , ▁sumatera ... (+4 more)` | 14 |
-| 64k | `▁natal ▁ima ▁sada ▁kecamatan ▁di ▁kabupaten ▁mandailing ▁natal , ▁sumatera ... (+4 more)` | 14 |
+| 8k | `▁kumpulan ▁set ia ▁ima ▁sala ▁sada ▁huta ▁na ▁adong ▁i ... (+14 more)` | 24 |
+| 16k | `▁kumpulan ▁setia ▁ima ▁sala ▁sada ▁huta ▁na ▁adong ▁i ▁kecamatan ... (+13 more)` | 23 |
+| 32k | `▁kumpulan ▁setia ▁ima ▁sala ▁sada ▁huta ▁na ▁adong ▁i ▁kecamatan ... (+13 more)` | 23 |
+| 64k | `▁kumpulan ▁setia ▁ima ▁sala ▁sada ▁huta ▁na ▁adong ▁i ▁kecamatan ... (+13 more)` | 23 |
 
-**Sample 2:** `Luak Kakuasoan ima luak karejo perangkat pamarentah pusat na mandalankon karejo ...`
+**Sample 2:** `Muara Soma ima sala sada huta na ading i kecamatan Batang Natal, kabupaten Manda...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁luak ▁kakuasoan ▁ima ▁luak ▁karejo ▁perangkat ▁pamarentah ▁pusat ▁na ▁mandalankon ... (+9 more)` | 19 |
-| 16k | `▁luak ▁kakuasoan ▁ima ▁luak ▁karejo ▁perangkat ▁pamarentah ▁pusat ▁na ▁mandalankon ... (+9 more)` | 19 |
-| 32k | `▁luak ▁kakuasoan ▁ima ▁luak ▁karejo ▁perangkat ▁pamarentah ▁pusat ▁na ▁mandalankon ... (+9 more)` | 19 |
-| 64k | `▁luak ▁kakuasoan ▁ima ▁luak ▁karejo ▁perangkat ▁pamarentah ▁pusat ▁na ▁mandalankon ... (+9 more)` | 19 |
+| 8k | `▁muara ▁so ma ▁ima ▁sala ▁sada ▁huta ▁na ▁ading ▁i ... (+14 more)` | 24 |
+| 16k | `▁muara ▁soma ▁ima ▁sala ▁sada ▁huta ▁na ▁ading ▁i ▁kecamatan ... (+13 more)` | 23 |
+| 32k | `▁muara ▁soma ▁ima ▁sala ▁sada ▁huta ▁na ▁ading ▁i ▁kecamatan ... (+13 more)` | 23 |
+| 64k | `▁muara ▁soma ▁ima ▁sala ▁sada ▁huta ▁na ▁ading ▁i ▁kecamatan ... (+13 more)` | 23 |
 
-**Sample 3:** `17 Juni' ima ari pa-169 (ari pa-170 i taon kabisat) i kalender Gregorian.`
+**Sample 3:** `24 Januari ima ari pa-24 i kalender Gregorian dohot 361 ari (sanga 362 ari i tao...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁ 1 7 ▁juni ' ▁ima ▁ari ▁pa - 1 ... (+17 more)` | 27 |
-| 16k | `▁ 1 7 ▁juni ' ▁ima ▁ari ▁pa - 1 ... (+17 more)` | 27 |
-| 32k | `▁ 1 7 ▁juni ' ▁ima ▁ari ▁pa - 1 ... (+17 more)` | 27 |
-| 64k | `▁ 1 7 ▁juni ' ▁ima ▁ari ▁pa - 1 ... (+17 more)` | 27 |
+| 8k | `▁ 2 4 ▁januari ▁ima ▁ari ▁pa - 2 4 ... (+24 more)` | 34 |
+| 16k | `▁ 2 4 ▁januari ▁ima ▁ari ▁pa - 2 4 ... (+24 more)` | 34 |
+| 32k | `▁ 2 4 ▁januari ▁ima ▁ari ▁pa - 2 4 ... (+24 more)` | 34 |
+| 64k | `▁ 2 4 ▁januari ▁ima ▁ari ▁pa - 2 4 ... (+24 more)` | 34 |
 
 
 ### Key Findings
 
 - **Best Compression:** 64k achieves 5.210x compression
-- **Lowest UNK Rate:** 8k with 0.0869% unknown tokens
+- **Lowest UNK Rate:** 8k with 0.0881% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 2,126 | 11.05 | 3,791 | 25.0% | 62.5% |
-| **2-gram** | Subword | 193 🏆 | 7.60 | 1,424 | 75.4% | 99.7% |
-| **3-gram** | Word | 1,572 | 10.62 | 2,726 | 28.6% | 65.6% |
-| **3-gram** | Subword | 1,481 | 10.53 | 9,264 | 32.5% | 79.4% |
-| **4-gram** | Word | 1,863 | 10.86 | 3,343 | 28.5% | 56.4% |
-| **4-gram** | Subword | 7,317 | 12.84 | 38,756 | 16.0% | 47.2% |
+| **2-gram** | Word | 2,149 | 11.07 | 3,846 | 24.9% | 62.3% |
+| **2-gram** | Subword | 193 🏆 | 7.59 | 1,424 | 75.5% | 99.7% |
+| **3-gram** | Word | 1,623 | 10.66 | 2,810 | 28.2% | 64.8% |
+| **3-gram** | Subword | 1,481 | 10.53 | 9,326 | 32.5% | 79.4% |
+| **4-gram** | Word | 1,998 | 10.96 | 3,539 | 27.5% | 54.8% |
+| **4-gram** | Subword | 7,322 | 12.84 | 39,044 | 16.0% | 47.2% |
+| **5-gram** | Word | 980 | 9.94 | 1,944 | 37.4% | 71.2% |
+| **5-gram** | Subword | 20,669 | 14.34 | 80,096 | 9.7% | 30.8% |
 
 ### Top 5 N-grams by Size
 
@@ -150,68 +162,88 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `ima sada` | 613 |
-| 2 | `on pe` | 485 |
-| 3 | `na adong` | 408 |
-| 4 | `sian on` | 359 |
-| 5 | `i taon` | 350 |
+| 1 | `ima sada` | 626 |
+| 2 | `on pe` | 512 |
+| 3 | `na adong` | 416 |
+| 4 | `sian on` | 373 |
+| 5 | `i taon` | 359 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `na adong i` | 259 |
-| 2 | `kabupaten mandailing natal` | 176 |
-| 3 | `i kalender gregorian` | 169 |
-| 4 | `ima ari pa` | 156 |
-| 5 | `sumatera utara indonesia` | 156 |
+| 1 | `na adong i` | 265 |
+| 2 | `kabupaten mandailing natal` | 178 |
+| 3 | `i kalender gregorian` | 170 |
+| 4 | `sumatera utara indonesia` | 160 |
+| 5 | `ima ari pa` | 157 |
 
 **4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `provinsi sumatera utara indonesia` | 130 |
-| 2 | `kabupaten mandailing natal provinsi` | 127 |
-| 3 | `natal provinsi sumatera utara` | 126 |
-| 4 | `mandailing natal provinsi sumatera` | 126 |
-| 5 | `taon kabisat i kalender` | 125 |
+| 1 | `provinsi sumatera utara indonesia` | 133 |
+| 2 | `kabupaten mandailing natal provinsi` | 130 |
+| 3 | `mandailing natal provinsi sumatera` | 129 |
+| 4 | `natal provinsi sumatera utara` | 129 |
+| 5 | `taon kabisat i kalender` | 126 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `kabupaten mandailing natal provinsi sumatera` | 129 |
+| 2 | `mandailing natal provinsi sumatera utara` | 129 |
+| 3 | `natal provinsi sumatera utara indonesia` | 128 |
+| 4 | `taon kabisat i kalender gregorian` | 126 |
+| 5 | `huta na adong i kecamatan` | 112 |
 
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `a n` | 41,122 |
-| 2 | `a _` | 36,766 |
-| 3 | `n _` | 28,003 |
-| 4 | `m a` | 25,432 |
-| 5 | `i _` | 24,703 |
+| 1 | `a n` | 41,734 |
+| 2 | `a _` | 37,272 |
+| 3 | `n _` | 28,447 |
+| 4 | `m a` | 25,826 |
+| 5 | `i _` | 25,144 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `_ m a` | 15,316 |
-| 2 | `a n _` | 13,300 |
-| 3 | `a n g` | 11,520 |
-| 4 | `_ n a` | 11,505 |
-| 5 | `n a _` | 10,547 |
+| 1 | `_ m a` | 15,579 |
+| 2 | `a n _` | 13,475 |
+| 3 | `_ n a` | 11,682 |
+| 4 | `a n g` | 11,673 |
+| 5 | `n a _` | 10,767 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `_ n a _` | 6,885 |
-| 2 | `_ m a n` | 5,972 |
-| 3 | `a _ m a` | 4,367 |
-| 4 | `i m a _` | 4,073 |
-| 5 | `_ i m a` | 4,072 |
+| 1 | `_ n a _` | 7,012 |
+| 2 | `_ m a n` | 6,102 |
+| 3 | `a _ m a` | 4,445 |
+| 4 | `_ i m a` | 4,125 |
+| 5 | `i m a _` | 4,121 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `_ i m a _` | 3,948 |
+| 2 | `d o h o t` | 3,004 |
+| 3 | `o h o t _` | 3,001 |
+| 4 | `_ d o h o` | 2,997 |
+| 5 | `_ d o t _` | 2,471 |
 
 
 ### Key Findings
 
 - **Best Perplexity:** 2-gram (subword) with 193
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~47% of corpus
+- **Coverage:** Top-1000 patterns cover ~31% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.8051 | 1.747 | 4.52 | 26,321 | 19.5% |
-| **1** | Subword | 0.8855 | 1.847 | 5.45 | 845 | 11.4% |
-| **2** | Word | 0.2150 | 1.161 | 1.41 | 118,363 | 78.5% |
-| **2** | Subword | 0.7881 | 1.727 | 4.37 | 4,600 | 21.2% |
-| **3** | Word | 0.0511 | 1.036 | 1.07 | 165,958 | 94.9% |
-| **3** | Subword | 0.7696 | 1.705 | 3.50 | 20,094 | 23.0% |
-| **4** | Word | 0.0119 🏆 | 1.008 | 1.01 | 176,808 | 98.8% |
-| **4** | Subword | 0.5810 | 1.496 | 2.41 | 70,348 | 41.9% |
+| **1** | Word | 0.8033 | 1.745 | 4.52 | 26,637 | 19.7% |
+| **1** | Subword | 0.8859 | 1.848 | 5.46 | 845 | 11.4% |
+| **2** | Word | 0.2155 | 1.161 | 1.41 | 119,766 | 78.4% |
+| **2** | Subword | 0.7876 | 1.726 | 4.38 | 4,613 | 21.2% |
+| **3** | Word | 0.0517 | 1.037 | 1.07 | 168,163 | 94.8% |
+| **3** | Subword | 0.7693 | 1.704 | 3.51 | 20,191 | 23.1% |
+| **4** | Word | 0.0122 🏆 | 1.008 | 1.02 | 179,311 | 98.8% |
+| **4** | Subword | 0.5814 | 1.496 | 2.41 | 70,850 | 41.9% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `i etika deskriptif tanpa jejak ni ibana pautan luar angkasa internasional dot filsafat tarbonggal im...`
-2. `na ma tu kadiua pengantin adaboru i kota nagodangna ima kalimat frasa arab ا alif alif`
-3. `ima simfoni dagdanak carito na adong i kamajuan sosial yang ditunjukkan dalam menjelaskan proses pal...`
+1. `i kota di kotu isa rupana kahanggi namar sisolkot ni eme ni awak dot mamakena pala`
+2. `na mandung manjadi aliran eksistensialisme sartre ima al qur an sm 180 an sm 70 an`
+3. `ima sada provinsi sumatera utara aek sasataon rodang momo tarida do anggina si baroar dibaon na`
 
 **Context Size 2:**
 
-1. `ima sada pamikir paling ponting ison ima bagain ni alak etika 24 25 manjadi cabang ni elmu`
-2. `on pe i artion ima panasehat mara boru na tobang tingon saro perancis partongaan dot pangujung abad`
-3. `na adong i harana suden aon bisa di turuti dungi anggon na idalani satiop get manyuan anso`
+1. `ima sada sunni mazhab hanafi vasilij vladimirovič bartold art by barbara brend p 130 tai ulama na`
+2. `on pe mandung dewasa pakean nai gunaon pakean adat belitong tai i instrospeksi eksperimental sudena ...`
+3. `na adong juo alak sunni dot 10 huruf ngolu vokal sapetona hangeul adongdope 3 konsonannai dot 1`
 
 **Context Size 3:**
 
-1. `na adong i mandailing ima ibagain jolo ni bagas on samuloi on toru sampe tu ginjang i jepang`
-2. `kabupaten mandailing natal sumatera utara indonesia baru koordinat nai ima na adong tingon simatoban...`
-3. `ima ari pa 105 ari pa 106 i taon kabisat i kalender gregorian dohot 361 ari sanga 362`
+1. `na adong i ruang woktu i sakitar lubang nalomlom adong parmukoan na i dokon horizon peristiwa objek ...`
+2. `kabupaten mandailing natal provinsi sumatera utara indonesia i botung adong luak parmayaman na deges...`
+3. `ima ari pa 103 ari pa 104 i taon kabisat i kalender gregorian dohot 363 ari sanga 364`
 
 **Context Size 4:**
 
 1. `kabupaten mandailing natal provinsi sumatera utara indonesia sumberna`
-2. `natal provinsi sumatera utara indonesia huta on pe adong na ima sacara alami do on inda na ibaen bae...`
-3. `mandailing natal provinsi sumatera utara indonesia i batahan`
+2. `natal provinsi sumatera utara indonesia pula sian on panyabungan tu kecamatan on`
+3. `mandailing natal provinsi sumatera utara indonesia sumberna`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `a_hur:_ig_bumani`
-2. `_i_a_0_a_u_agong`
-3. `numayalleri_dusi`
+1. `alan_a_rian_ruse`
+2. `_ana_ontuon._tan`
+3. `nang_akeon_asapa`
 
 **Context Size 2:**
 
-1. `ang_rovskithe_tin`
-2. `a_tikabindot_puna`
-3. `n_nak,_ina_dohorc`
+1. `an_niviusi,_hamel`
+2. `a_ida_lak_nai_jun`
+3. `n_sentat_dokon_ng`
 
 **Context Size 3:**
 
-1. `_man_baru_najo._am`
-2. `an_reicht_ditasali`
-3. `ang_i_the_pada_raj`
+1. `_mambaen_dohot_par`
+2. `an_ibad_oktu_piga_`
+3. `_nagoda_marcoundur`
 
 **Context Size 4:**
 
-1. `_na_mander_gregoria`
-2. `_manurutnia_iangir_`
-3. `a_mang,_31_taon_ima`
+1. `_na_ibaen_herito_la`
+2. `_manjadi_i_ruar_tu_`
+3. `a_marisi.dw:_menek_`
 
 
 ### Key Findings
 
 - **Best Predictability:** Context-4 (word) with 98.8% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (70,348 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (70,850 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,64 +346,64 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 11,024 |
-| Total Tokens | 173,772 |
-| Mean Frequency | 15.76 |
+| Vocabulary Size | 11,148 |
+| Total Tokens | 176,428 |
+| Mean Frequency | 15.83 |
 | Median Frequency | 4 |
-| Frequency Std Dev | 129.04 |
+| Frequency Std Dev | 130.57 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | i | 7,102 |
-| 2 | na | 6,996 |
-| 3 | ima | 3,950 |
-| 4 | on | 3,907 |
-| 5 | dohot | 2,932 |
-| 6 | ni | 2,627 |
-| 7 | dot | 2,463 |
-| 8 | sada | 1,805 |
-| 9 | tu | 1,679 |
-| 10 | ma | 1,474 |
+| 1 | i | 7,229 |
+| 2 | na | 7,125 |
+| 3 | on | 3,997 |
+| 4 | ima | 3,996 |
+| 5 | dohot | 2,990 |
+| 6 | ni | 2,685 |
+| 7 | dot | 2,484 |
+| 8 | sada | 1,834 |
+| 9 | tu | 1,711 |
+| 10 | ma | 1,485 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | harvard | 2 |
-| 2 | syahadat | 2 |
-| 3 | dans | 2 |
-| 4 | philosophie | 2 |
-| 5 | évasion | 2 |
-| 6 | bénézé | 2 |
-| 7 | infini | 2 |
-| 8 | delà | 2 |
-| 9 | telos | 2 |
-| 10 | apganistan | 2 |
+| 1 | lil | 2 |
+| 2 | imah | 2 |
+| 3 | nasida | 2 |
+| 4 | sunusi | 2 |
+| 5 | nunga | 2 |
+| 6 | majmu | 2 |
+| 7 | fatawa | 2 |
+| 8 | fiqhi | 2 |
+| 9 | panjalakian | 2 |
+| 10 | martoba | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 1.0692 |
-| R² (Goodness of Fit) | 0.988968 |
+| Zipf Coefficient | 1.0705 |
+| R² (Goodness of Fit) | 0.989075 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 41.7% |
+| Top 100 | 41.8% |
 | Top 1,000 | 71.1% |
-| Top 5,000 | 91.5% |
-| Top 10,000 | 98.8% |
+| Top 5,000 | 91.4% |
+| Top 10,000 | 98.7% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9890 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 41.7% of corpus
-- **Long Tail:** 1,024 words needed for remaining 1.2% coverage
+- **Zipf Compliance:** R²=0.9891 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 41.8% of corpus
+- **Long Tail:** 1,148 words needed for remaining 1.3% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.3926 🏆 | 0.4276 | N/A | N/A |
-| **mono_64d** | 64 | 0.1169 | 0.4242 | N/A | N/A |
-| **mono_128d** | 128 | 0.0230 | 0.4239 | N/A | N/A |
+| **mono_32d** | 32 | 0.4518 🏆 | 0.4274 | N/A | N/A |
+| **mono_64d** | 64 | 0.1211 | 0.4252 | N/A | N/A |
+| **mono_128d** | 128 | 0.0249 | 0.4089 | N/A | N/A |
+| **aligned_32d** | 32 | 0.4518 | 0.4145 | 0.0140 | 0.1240 |
+| **aligned_64d** | 64 | 0.1211 | 0.4363 | 0.0200 | 0.1760 |
+| **aligned_128d** | 128 | 0.0249 | 0.4097 | 0.0540 | 0.2300 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.3926 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.4252. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** mono_32d with 0.4518 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.4203. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 5.4% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **1.311** | High formulaic/idiomatic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -426,25 +461,23 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Prefixes
 | Prefix | Examples |
 |--------|----------|
-| `-ma` | marmangan, manjadion, manembak |
-| `-pa` | palo, pasiap, panyalahgunaan |
-| `-man` | manjadion, manembak, mangargai |
-| `-mar` | marmangan, mardikir, markombang |
-| `-sa` | salama, samo, sasabagas |
-| `-ta` | tagalog, tas, targinjang |
-| `-ka` | karang, kadua, kamis |
+| `-ma` | marmasak, mamuloi, maligina |
+| `-pa` | paderi, parkumpulan, pangajaran |
+| `-man` | manakik, manyorang, mangajari |
+| `-mar` | marmasak, marwujud, mariner |
+| `-sa` | samananjung, sati, sakral |
+| `-ta` | tarpusat, takar, tajikistan |
 
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-n` | asisten, tolongan, proclamation |
-| `-an` | tolongan, panyalahgunaan, marmangan |
-| `-a` | tionghua, natarida, moskwa |
-| `-ng` | karang, gedung, targinjang |
-| `-on` | proclamation, idasorkon, manjadion |
-| `-na` | pascasarjana, nalainna, paduana |
-| `-ang` | karang, targinjang, uwang |
-| `-kon` | idasorkon, ilaporkon, namangobankon |
+| `-n` | tubagasan, ringkasan, disusun |
+| `-a` | nikola, studia, katua |
+| `-an` | tubagasan, ringkasan, parkumpulan |
+| `-ng` | samananjung, pedagang, kacang |
+| `-on` | bandingkon, dibandingkon, pelestarion |
+| `-na` | maligina, umurna, ajayaanna |
+| `-ang` | pedagang, kacang, sumbayang |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -452,18 +485,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `anga` | 1.50x | 76 contexts | nanga, angan, sanga |
-| `angk` | 1.52x | 58 contexts | angke, angka, angko |
-| `mang` | 1.68x | 31 contexts | amang, mango, tamang |
-| `anda` | 1.40x | 53 contexts | tanda, banda, ganda |
-| `dang` | 1.48x | 41 contexts | udang, undang, sedang |
-| `amba` | 1.48x | 39 contexts | hamba, tamba, gambar |
-| `aran` | 1.38x | 47 contexts | arani, arang, arana |
-| `ngka` | 1.41x | 39 contexts | angka, dangka, angkat |
-| `ngan` | 1.32x | 43 contexts | angan, tangan, lengan |
-| `anja` | 1.38x | 33 contexts | hanja, banjar, anjadi |
-| `angg` | 1.31x | 39 contexts | anggi, anggo, anggap |
-| `tang` | 1.35x | 29 contexts | utang, otang, tangan |
+| `anga` | 1.46x | 77 contexts | nanga, angan, sanga |
+| `angk` | 1.47x | 58 contexts | angko, angke, angka |
+| `anda` | 1.43x | 54 contexts | ganda, tanda, banda |
+| `mang` | 1.59x | 31 contexts | mango, amang, lomang |
+| `amba` | 1.49x | 39 contexts | hamba, tamba, sambal |
+| `ngan` | 1.40x | 43 contexts | angan, lengan, sangan |
+| `dang` | 1.40x | 42 contexts | udang, ndang, dangka |
+| `aran` | 1.35x | 48 contexts | arana, arang, saran |
+| `angg` | 1.32x | 39 contexts | anggi, anggo, nangge |
+| `anja` | 1.36x | 34 contexts | hanja, banjar, anjadi |
+| `ngga` | 1.37x | 30 contexts | hingga, rongga, mangga |
+| `ting` | 1.34x | 32 contexts | tingo, uting, tingon |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -471,16 +504,16 @@ This table shows which prefixes and suffixes most frequently co-occur on the sam
 
 | Prefix | Suffix | Frequency | Examples |
 |--------|--------|-----------|----------|
-| `-pa` | `-n` | 297 words | parsiajaran, paridian |
-| `-pa` | `-an` | 266 words | parsiajaran, paridian |
-| `-ma` | `-n` | 232 words | malainkon, malahirkon |
-| `-ma` | `-on` | 148 words | malainkon, malahirkon |
-| `-ka` | `-n` | 111 words | kabupaten, kahangatan |
-| `-ka` | `-an` | 108 words | kahangatan, kapastian |
-| `-ma` | `-kon` | 96 words | malainkon, malahirkon |
-| `-ma` | `-a` | 96 words | marga, maninggalnaia |
-| `-ma` | `-ng` | 67 words | markombang, margelombang |
-| `-ma` | `-an` | 60 words | marsegaan, masakan |
+| `-pa` | `-n` | 307 words | panjalakan, pambaenan |
+| `-pa` | `-an` | 271 words | panjalakan, pambaenan |
+| `-ma` | `-n` | 241 words | mangombangkon, maximilian |
+| `-ma` | `-on` | 157 words | mangombangkon, manyesuaion |
+| `-ma` | `-a` | 98 words | maringana, manurutnia |
+| `-ma` | `-ng` | 69 words | malang, marancang |
+| `-ma` | `-an` | 61 words | maximilian, marhalangan |
+| `-pa` | `-a` | 57 words | pasca, pasadana |
+| `-sa` | `-a` | 40 words | samentara, sangapiga |
+| `-ma` | `-ang` | 38 words | malang, marancang |
 
 ### 6.5 Recursive Morpheme Segmentation
 
@@ -488,26 +521,28 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
-| markalanjutan | **`mar-ka-lanjut-an`** | 7.5 | `lanjut` |
-| malambangkon | **`ma-lamb-ang-kon`** | 7.5 | `lamb` |
-| kabolakangan | **`ka-bolak-ang-an`** | 7.5 | `bolak` |
-| kamanusiaan | **`ka-man-usia-an`** | 7.5 | `usia` |
-| kakuasoanna | **`ka-kuaso-an-na`** | 7.5 | `kuaso` |
-| markabangsoan | **`mar-ka-bangso-an`** | 7.5 | `bangso` |
-| sakaturunan | **`sa-ka-turun-an`** | 7.5 | `turun` |
-| pamabangan | **`pa-ma-bang-an`** | 7.5 | `bang` |
 | paporangan | **`pa-pora-ng-an`** | 7.5 | `pora` |
-| kaputusan | **`ka-putus-an`** | 6.0 | `putus` |
-| martibalna | **`mar-tibal-na`** | 6.0 | `tibal` |
-| mandapatkon | **`man-dapat-kon`** | 6.0 | `dapat` |
-| kaseharian | **`ka-sehari-an`** | 6.0 | `sehari` |
-| malahirkon | **`ma-lahir-kon`** | 6.0 | `lahir` |
-| kayakinan | **`ka-yakin-an`** | 6.0 | `yakin` |
+| marpandangan | **`mar-pa-ndang-an`** | 7.5 | `ndang` |
+| bagasanna | **`bagas-an-na`** | 6.0 | `bagas` |
+| pasabolas | **`pa-sa-bolas`** | 6.0 | `bolas` |
+| mandurung | **`man-duru-ng`** | 6.0 | `duru` |
+| sasabagas | **`sa-sa-bagas`** | 6.0 | `bagas` |
+| sabalikna | **`sa-balik-na`** | 6.0 | `balik` |
+| marlainan | **`mar-lain-an`** | 6.0 | `lain` |
+| panilaian | **`pa-nilai-an`** | 6.0 | `nilai` |
+| mardongan | **`mar-dong-an`** | 6.0 | `dong` |
+| margontian | **`mar-gonti-an`** | 6.0 | `gonti` |
+| mandefinision | **`man-definisi-on`** | 6.0 | `definisi` |
+| pemerintahan | **`pemerintah-an`** | 4.5 | `pemerintah` |
+| margandak | **`mar-gandak`** | 4.5 | `gandak` |
+| habitatna | **`habitat-na`** | 4.5 | `habitat` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language BTM appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Batak Mandailing shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
+
+> **Note on Idiomaticity:** The high Idiomaticity Gap suggests a large number of frequent multi-word expressions or formulaic sequences that are statistically distinct from their component parts.
 
 ---
 ## 7. Summary & Recommendations
@@ -734,4 +769,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 08:51:39*
+*Report Date: 2026-01-03 19:44:07*

models/embeddings/aligned/btm_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7e3fef86fa21d06d0ce4b31e1fa7d94c30d5ae1821449e0030b3ff6872dc30b
+size 1028713761

models/embeddings/aligned/btm_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "btm", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/btm_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06d6fdcb59ad3528d2f3dad8b44ad23ecaaca8606a93fbc996126606d739ec9d
+size 65664

models/embeddings/aligned/btm_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "btm",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 1376,
+  "vocab_size": 4531
+}

models/embeddings/aligned/btm_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f3c3ec079508e79445feac4bc2fcddf0e842bf5419162533c21370ee7149a46
+size 257233953

models/embeddings/aligned/btm_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "btm", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/btm_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3c44b4bec5ed198762493c46960bd6b7a54fc97208b663f55266513562c9995
+size 4224

models/embeddings/aligned/btm_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "btm",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 1376,
+  "vocab_size": 4531
+}

models/embeddings/aligned/btm_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc52d868e9411740247d4dd6f170e2856ac55de2677659f501a05727c06ce113
+size 514393889

models/embeddings/aligned/btm_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "btm", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/btm_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c4628ab8e46fcb3dd663797d8a49949b168e697caa63f8542c4d151c0bcab06
+size 16512

models/embeddings/aligned/btm_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "btm",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 1376,
+  "vocab_size": 4531
+}

models/embeddings/monolingual/btm_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eab8193f169ed951260bc5e2a8335421cc96bdc9afa7a4fbc2e5f19107d9f18c
-size 1028667993
+oid sha256:d7e3fef86fa21d06d0ce4b31e1fa7d94c30d5ae1821449e0030b3ff6872dc30b
+size 1028713761

models/embeddings/monolingual/btm_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 4487
+  "vocab_size": 4531
 }

models/embeddings/monolingual/btm_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c0f87796e30b250180cd4e0a21e5c53df57dbf3ac48c6ece59a24ce9ea2a2f9
-size 257221977
+oid sha256:9f3c3ec079508e79445feac4bc2fcddf0e842bf5419162533c21370ee7149a46
+size 257233953

models/embeddings/monolingual/btm_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 4487
+  "vocab_size": 4531
 }

models/embeddings/monolingual/btm_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:06a10d115cb9f5acc6300b8c1370ad70ca5bbbf741198c826bb6b2380bd77f72
-size 514370649
+oid sha256:dc52d868e9411740247d4dd6f170e2856ac55de2677659f501a05727c06ce113
+size 514393889

models/embeddings/monolingual/btm_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 4487
+  "vocab_size": 4531
 }

models/subword_markov/btm_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a5a6ceefeddffa92ade2020283b0670a56c9de4e981d4c37d567af5d7edf00c
-size 40549
+oid sha256:7ca0b5561b51295ee9176d57fc0e034e7a9025e7d704682ec8b1c69163c201ca
+size 40526

models/subword_markov/btm_markov_ctx1_subword_metadata.json

@@ -3,5 +3,5 @@
   "variant": "subword",
   "language": "btm",
   "unique_contexts": 845,
-  "total_transitions": 1194771
+  "total_transitions": 1212124
 }

models/subword_markov/btm_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8e63c72b397369614152fea75601c14782ddf5f412d72b019eb11cd857384b5
-size 173900
+oid sha256:a41892dc03807b9211991f8cbd56b67ea22d11eca3fbb7a1fb84a184dfdbd050
+size 174290

models/subword_markov/btm_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "btm",
-  "unique_contexts": 4600,
-  "total_transitions": 1193471
+  "unique_contexts": 4613,
+  "total_transitions": 1210807
 }

models/subword_markov/btm_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b7597ab768a07a1247aa2becf4d6d80afbba74e537d60f41be994856978e37bc
-size 548580
+oid sha256:3548eb74171bd17f30e9b9daed8167a9366cbfaf6040d4b9856fe5adfe305353
+size 554127

models/subword_markov/btm_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "btm",
-  "unique_contexts": 20094,
-  "total_transitions": 1192171
+  "unique_contexts": 20191,
+  "total_transitions": 1209490
 }

models/subword_markov/btm_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4520c606c01af7c8dd7bdc888cc9cc1db3d4aa76abe56123d646a75bf737e6d
-size 1327069
+oid sha256:dc7cda5e3c90773acfe7a97989454135215708f07d6d99bf279499d5ee0cab02
+size 1338222

models/subword_markov/btm_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "btm",
-  "unique_contexts": 70348,
-  "total_transitions": 1190871
+  "unique_contexts": 70850,
+  "total_transitions": 1208173
 }

models/subword_ngram/btm_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2df596ff923d644e93e8a0534100541ef70c30a47243fc6d946a28b41b0b391f
-size 19810
+oid sha256:a0fdd1191205a4def7e66cf30101a732f47b5dac125c03d75624e856c019c312
+size 19955

models/subword_ngram/btm_2gram_subword_metadata.json

@@ -3,5 +3,5 @@
   "variant": "subword",
   "language": "btm",
   "unique_ngrams": 1424,
-  "total_ngrams": 1194771
+  "total_ngrams": 1212124
 }

models/subword_ngram/btm_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c67b8a177fdee1d2cae2461ece233e72cc6271f2ec127adb17efbab362cb96f
-size 106854
+oid sha256:3a1a537d3cc81d4e143b205d0e87fbd2fdd549393281217db048fc77e59892d0
+size 107259

models/subword_ngram/btm_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "btm",
-  "unique_ngrams": 9264,
-  "total_ngrams": 1193471
+  "unique_ngrams": 9326,
+  "total_ngrams": 1210807
 }

models/subword_ngram/btm_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:685a19fd481b30cfe583384638a09926eca754668b4379cc350384ca3d6ec64d
-size 469711
+oid sha256:80779bbebe1289cb03ff754cced84699a85f992607839a4d892da17b75f17e57
+size 475626

models/subword_ngram/btm_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "btm",
-  "unique_ngrams": 38756,
-  "total_ngrams": 1192171
+  "unique_ngrams": 39044,
+  "total_ngrams": 1209490
 }

models/subword_ngram/btm_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:639a83c77f836f4b29fd645fc41318b2d1ea5d6c7226276f867be054f14e6bbe
+size 937142

models/subword_ngram/btm_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "btm",
+  "unique_ngrams": 80096,
+  "total_ngrams": 1208173
+}

models/tokenizer/btm_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c56b4bf3a440a54cda19a21cb12647b4eda168f5ed41c73de87c5dab1bb403c
-size 504094
+oid sha256:e461e5ef3a1cc4c62b47db593915a7f606a17f2e1f1327e3fce0974e4fb5226f
+size 504243

models/tokenizer/btm_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f9e24ab010b800e8b37dbe7ba7c3683db68cd1f831fc7c1773820eddf148959
-size 792383
+oid sha256:e66cabc6e1fdba0f68828481107910af32adda8afad30aed4f5608a81fb4a97b
+size 792574

models/tokenizer/btm_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e66c10c510f5dcaba827e3e9710ee5e2bf9168b1de095a4d7f971bdd4106bdae
-size 1307068
+oid sha256:03a65a40f671f902121336bf18af8dbea088e84ffb9b86f2a163be3f189d9fac
+size 1307737

models/tokenizer/btm_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9a7e75986934266a99f2c51f3d64b3ab75a0c504b45e45c89b6718cdc0aa876
-size 372323
+oid sha256:eef49f14a8d30d9a231e33e3b610ca71c7e55b5b5c49383865879ce21bca5462
+size 372365

models/vocabulary/btm_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bbd9b813ffcb4592d15089be87b44c35ccb530113e5f1cdc05f8deb485295f2d
-size 177391
+oid sha256:89d8c3b3f96ce2dfaed27e33f252f3bb19087fa67c088e6a39840417eb43b019
+size 179833

models/vocabulary/btm_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "btm",
-  "vocabulary_size": 11024,
+  "vocabulary_size": 11148,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.1393775944581053,
+    "type_token_ratio": 0.1389514045177961,
     "coverage": {
-      "top_100": 0.3835752624203485,
-      "top_1000": 0.6533618888977023,
-      "top_5000": 0.8406599508209724,
-      "top_10000": 0.9080880992041458
+      "top_100": 0.3839449445694757,
+      "top_1000": 0.6534237726098191,
+      "top_5000": 0.8400589730766025,
+      "top_10000": 0.9071642910727682
     },
-    "hapax_count": 15333,
-    "hapax_ratio": 0.5817429904769131,
-    "total_documents": 1300
+    "hapax_count": 15524,
+    "hapax_ratio": 0.5820335932813437,
+    "total_documents": 1317
   }
 }

models/word_markov/btm_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7836709a62d3e6d6f77608b52ac844f6dc543e833b71cee1fecb9081e7920a1
-size 896043
+oid sha256:7c9e61f4eac88266a14896004b747b93aea2b8270214b2b9f82d2cbdf767d220
+size 906482

models/word_markov/btm_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "btm",
-  "unique_contexts": 26321,
-  "total_transitions": 187805
+  "unique_contexts": 26637,
+  "total_transitions": 190635
 }

models/word_markov/btm_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e923879187734024aa16a8259d19e5aa01d660c5479ca45138d2389819ed17a8
-size 2002914
+oid sha256:f05642bdaa511b298f96adf1f87dde7a273920e3f61de6e9260250e0957c3ba3
+size 2027650

models/word_markov/btm_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "btm",
-  "unique_contexts": 118363,
-  "total_transitions": 186505
+  "unique_contexts": 119766,
+  "total_transitions": 189318
 }