Upload all models and assets for bjn (latest)

.gitattributes

@@ -38,3 +38,4 @@ visualizations/performance_dashboard.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: bjn
-language_name: BJN
+language_name: Banjar
 language_family: austronesian_malay
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-austronesian_malay
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.828
+    value: 4.830
   - name: best_isotropy
     type: isotropy
-    value: 0.8673
+    value: 0.8715
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# BJN - Wikilangs Models
+# Banjar - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **BJN** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Banjar** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.761x | 3.76 | 0.3901% | 369,664 |
-| **16k** | 4.163x | 4.17 | 0.4318% | 333,925 |
-| **32k** | 4.537x | 4.54 | 0.4706% | 306,394 |
-| **64k** | 4.828x 🏆 | 4.83 | 0.5008% | 287,926 |
+| **8k** | 3.761x | 3.76 | 0.3950% | 367,048 |
+| **16k** | 4.164x | 4.17 | 0.4374% | 331,539 |
+| **32k** | 4.537x | 4.54 | 0.4766% | 304,229 |
+| **64k** | 4.830x 🏆 | 4.83 | 0.5073% | 285,820 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Gunung Anyar adalah sabuting kacamatan di Kuta Surabaya, Prupinsi Jawa Timur, In...`
+**Sample 1:** `Wedoro adalah sabuah kampung di Kacamatan Glagah, Kabupatin Lamongan, Prupinsi J...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁gunung ▁anyar ▁adalah ▁sabuting ▁kacamatan ▁di ▁kuta ▁surabaya , ▁prupinsi ... (+6 more)` | 16 |
-| 16k | `▁gunung ▁anyar ▁adalah ▁sabuting ▁kacamatan ▁di ▁kuta ▁surabaya , ▁prupinsi ... (+6 more)` | 16 |
-| 32k | `▁gunung ▁anyar ▁adalah ▁sabuting ▁kacamatan ▁di ▁kuta ▁surabaya , ▁prupinsi ... (+6 more)` | 16 |
-| 64k | `▁gunung ▁anyar ▁adalah ▁sabuting ▁kacamatan ▁di ▁kuta ▁surabaya , ▁prupinsi ... (+6 more)` | 16 |
+| 8k | `▁w ed oro ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁glagah , ... (+9 more)` | 19 |
+| 16k | `▁wed oro ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁glagah , ▁kabupatin ... (+8 more)` | 18 |
+| 32k | `▁wedoro ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁glagah , ▁kabupatin ▁lamongan ... (+7 more)` | 17 |
+| 64k | `▁wedoro ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁glagah , ▁kabupatin ▁lamongan ... (+7 more)` | 17 |
 
-**Sample 2:** `Tebedak adalah sabuah kampung di Kacamatan Ngabang, Kabupatin Landak, Prupinsi K...`
+**Sample 2:** `Laburan Baru' adalah sabuah kampung di Kacamatan Paser Belengkong, Kabupatin Pas...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁teb ed ak ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁ngabang , ... (+9 more)` | 19 |
-| 16k | `▁teb edak ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁ngabang , ▁kabupatin ... (+8 more)` | 18 |
-| 32k | `▁teb edak ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁ngabang , ▁kabupatin ... (+8 more)` | 18 |
-| 64k | `▁tebedak ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁ngabang , ▁kabupatin ▁landak ... (+7 more)` | 17 |
+| 8k | `▁lab uran ▁baru ' ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁paser ... (+12 more)` | 22 |
+| 16k | `▁lab uran ▁baru ' ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁paser ... (+11 more)` | 21 |
+| 32k | `▁laburan ▁baru ' ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁paser ▁belengkong ... (+10 more)` | 20 |
+| 64k | `▁laburan ▁baru ' ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁paser ▁belengkong ... (+10 more)` | 20 |
 
-**Sample 3:** `Handil Birayang Atas yaitu sabuting kampung di Kacamatan Bumi Makmur, Kabupatin ...`
+**Sample 3:** `Nibung adalah sabuah kampung di Kacamatan Selimbau, Kabupatin Kapuas Hulu, Prupi...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁handil ▁bir ayang ▁atas ▁yaitu ▁sabuting ▁kampung ▁di ▁kacamatan ▁bumi ... (+12 more)` | 22 |
-| 16k | `▁handil ▁bir ayang ▁atas ▁yaitu ▁sabuting ▁kampung ▁di ▁kacamatan ▁bumi ... (+12 more)` | 22 |
-| 32k | `▁handil ▁birayang ▁atas ▁yaitu ▁sabuting ▁kampung ▁di ▁kacamatan ▁bumi ▁makmur ... (+11 more)` | 21 |
-| 64k | `▁handil ▁birayang ▁atas ▁yaitu ▁sabuting ▁kampung ▁di ▁kacamatan ▁bumi ▁makmur ... (+11 more)` | 21 |
+| 8k | `▁n ib ung ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁sel imb ... (+12 more)` | 22 |
+| 16k | `▁n ibung ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁selimbau , ▁kabupatin ... (+9 more)` | 19 |
+| 32k | `▁nibung ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁selimbau , ▁kabupatin ▁kapuas ... (+8 more)` | 18 |
+| 64k | `▁nibung ▁adalah ▁sabuah ▁kampung ▁di ▁kacamatan ▁selimbau , ▁kabupatin ▁kapuas ... (+8 more)` | 18 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.828x compression
-- **Lowest UNK Rate:** 8k with 0.3901% unknown tokens
+- **Best Compression:** 64k achieves 4.830x compression
+- **Lowest UNK Rate:** 8k with 0.3950% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 6,379 | 12.64 | 21,446 | 23.8% | 44.7% |
-| **2-gram** | Subword | 186 🏆 | 7.54 | 2,773 | 78.2% | 99.5% |
-| **3-gram** | Word | 3,749 | 11.87 | 17,540 | 32.7% | 52.0% |
-| **3-gram** | Subword | 1,430 | 10.48 | 20,264 | 34.4% | 80.3% |
-| **4-gram** | Word | 5,222 | 12.35 | 24,563 | 29.0% | 48.3% |
-| **4-gram** | Subword | 7,617 | 12.90 | 99,301 | 17.6% | 50.0% |
+| **2-gram** | Word | 6,505 | 12.67 | 21,751 | 23.6% | 44.5% |
+| **2-gram** | Subword | 185 🏆 | 7.53 | 2,788 | 78.2% | 99.5% |
+| **3-gram** | Word | 3,849 | 11.91 | 17,881 | 32.5% | 51.6% |
+| **3-gram** | Subword | 1,428 | 10.48 | 20,293 | 34.4% | 80.3% |
+| **4-gram** | Word | 5,302 | 12.37 | 24,831 | 28.9% | 48.0% |
+| **4-gram** | Subword | 7,612 | 12.89 | 99,642 | 17.5% | 50.0% |
+| **5-gram** | Word | 4,712 | 12.20 | 16,656 | 25.7% | 48.4% |
+| **5-gram** | Subword | 25,009 | 14.61 | 245,459 | 12.3% | 34.0% |
 
 ### Top 5 N-grams by Size
 
@@ -150,68 +162,88 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `kampung di` | 5,958 |
-| 2 | `prupinsi kalimantan` | 5,878 |
-| 3 | `di kacamatan` | 5,619 |
-| 4 | `adalah sabuah` | 4,219 |
-| 5 | `sabuah kampung` | 3,812 |
+| 1 | `kampung di` | 5,961 |
+| 2 | `prupinsi kalimantan` | 5,903 |
+| 3 | `di kacamatan` | 5,625 |
+| 4 | `adalah sabuah` | 4,211 |
+| 5 | `sabuah kampung` | 3,806 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
 | 1 | `kampung di kacamatan` | 5,212 |
-| 2 | `adalah sabuah kampung` | 3,811 |
-| 3 | `sabuah kampung di` | 3,810 |
-| 4 | `kalimantan selatan indunisia` | 2,206 |
-| 5 | `prupinsi kalimantan selatan` | 2,192 |
+| 2 | `sabuah kampung di` | 3,803 |
+| 3 | `adalah sabuah kampung` | 3,803 |
+| 4 | `kalimantan selatan indunisia` | 2,201 |
+| 5 | `prupinsi kalimantan selatan` | 2,188 |
 
 **4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `sabuah kampung di kacamatan` | 3,810 |
-| 2 | `adalah sabuah kampung di` | 3,809 |
-| 3 | `prupinsi kalimantan selatan indunisia` | 2,158 |
+| 1 | `sabuah kampung di kacamatan` | 3,802 |
+| 2 | `adalah sabuah kampung di` | 3,801 |
+| 3 | `prupinsi kalimantan selatan indunisia` | 2,154 |
 | 4 | `prupinsi kalimantan barat indunisia` | 1,806 |
-| 5 | `sabuting kampung di kacamatan` | 1,342 |
+| 5 | `yaitu sabuting kampung di` | 1,356 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `adalah sabuah kampung di kacamatan` | 3,801 |
+| 2 | `yaitu sabuting kampung di kacamatan` | 1,253 |
+| 3 | `indunisia géografi watas wilayah watas` | 1,113 |
+| 4 | `géografi watas wilayah watas wilayah` | 1,099 |
+| 5 | `watas wilayah watas wilayah kacamatan` | 739 |
 
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `a n` | 360,335 |
-| 2 | `n _` | 192,183 |
-| 3 | `n g` | 150,787 |
-| 4 | `a _` | 137,197 |
-| 5 | `k a` | 130,918 |
+| 1 | `a n` | 365,243 |
+| 2 | `n _` | 194,875 |
+| 3 | `n g` | 152,971 |
+| 4 | `a _` | 138,836 |
+| 5 | `k a` | 132,349 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `a n _` | 154,048 |
-| 2 | `a n g` | 83,639 |
-| 3 | `_ k a` | 75,768 |
-| 4 | `n g _` | 74,596 |
-| 5 | `_ m a` | 57,180 |
+| 1 | `a n _` | 156,222 |
+| 2 | `a n g` | 84,871 |
+| 3 | `_ k a` | 76,502 |
+| 4 | `n g _` | 75,610 |
+| 5 | `_ m a` | 57,961 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `a n g _` | 48,221 |
-| 2 | `t a n _` | 34,250 |
-| 3 | `n a n g` | 29,540 |
-| 4 | `a t a n` | 29,135 |
-| 5 | `_ n a n` | 28,221 |
+| 1 | `a n g _` | 48,934 |
+| 2 | `t a n _` | 34,621 |
+| 3 | `n a n g` | 29,979 |
+| 4 | `a t a n` | 29,470 |
+| 5 | `_ n a n` | 28,658 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `_ n a n g` | 28,407 |
+| 2 | `n a n g _` | 27,864 |
+| 3 | `a t a n _` | 22,485 |
+| 4 | `m a t a n` | 17,997 |
+| 5 | `_ w a n _` | 17,178 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram (subword) with 186
+- **Best Perplexity:** 2-gram (subword) with 185
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~50% of corpus
+- **Coverage:** Top-1000 patterns cover ~34% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.8466 | 1.798 | 5.65 | 98,412 | 15.3% |
-| **1** | Subword | 0.7154 | 1.642 | 4.59 | 2,413 | 28.5% |
-| **2** | Word | 0.2332 | 1.175 | 1.48 | 553,823 | 76.7% |
-| **2** | Subword | 0.6822 | 1.605 | 4.16 | 11,071 | 31.8% |
-| **3** | Word | 0.0559 | 1.040 | 1.08 | 814,594 | 94.4% |
-| **3** | Subword | 0.7802 | 1.717 | 3.89 | 46,029 | 22.0% |
-| **4** | Word | 0.0149 🏆 | 1.010 | 1.02 | 878,428 | 98.5% |
-| **4** | Subword | 0.6536 | 1.573 | 2.81 | 179,116 | 34.6% |
+| **1** | Word | 0.8469 | 1.799 | 5.67 | 99,056 | 15.3% |
+| **1** | Subword | 0.7172 | 1.644 | 4.59 | 2,416 | 28.3% |
+| **2** | Word | 0.2337 | 1.176 | 1.48 | 559,810 | 76.6% |
+| **2** | Subword | 0.6823 | 1.605 | 4.16 | 11,092 | 31.8% |
+| **3** | Word | 0.0561 | 1.040 | 1.08 | 824,984 | 94.4% |
+| **3** | Subword | 0.7802 | 1.717 | 3.90 | 46,118 | 22.0% |
+| **4** | Word | 0.0150 🏆 | 1.010 | 1.02 | 890,043 | 98.5% |
+| **4** | Subword | 0.6544 | 1.574 | 2.81 | 179,736 | 34.6% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `di sarawak ihwal nitu pamimpin national league soccer wan upananda diikat albumin salain satelitnya ...`
-2. `nang kaya talabang pinggir papila yaitu hutan hujan pada sidang agung nang disuruh rayi rayiading ad...`
-3. `wan teolog kadada dana saganal ganalnya ujak langsung maka kata lawan kakawanannya sesama youtuber p...`
+1. `di kacamatan konang kabupatin sanggau prupinsi kalimantan tengah mesir india indunisia watas wilayah...`
+2. `nang baisi banyak banar dalam bahasa utama liga 3 m 1 sampai pamulaan wan takananya barupa`
+3. `wan manangani kajahatan gasan hintalu diploid buhannya kawa jua gasan pahitungan hisab nitu angin tu...`
 
 **Context Size 2:**
 
-1. `kampung di wilayah kacamatan sei menggaris pambagian administratip kacamatan batu ampar pambagian ad...`
-2. `prupinsi kalimantan barat indunisia géografi watas wilayah kampung baru yaitu sabuting kampung di ka...`
-3. `di kacamatan gambut kabupatin banjar prupinsi kalimantan timur indunisia géografi watas wilayah wata...`
+1. `kampung di kacamatan teluk sampit pambagian administratip kacamatan tualan hulu pambagian administra...`
+2. `prupinsi kalimantan timur indunisia makanan nangkaya tempe matan kacang kacangan imbah disangrai bad...`
+3. `di kacamatan menyuke kabupatin landak prupinsi kalimantan barat indunisia géografi watas wilayah kac...`
 
 **Context Size 3:**
 
-1. `kampung di kacamatan babat kabupatin lamongan prupinsi jawa timur indunisia jujuhutan`
-2. `adalah sabuah kampung di kacamatan parindu kabupatin sanggau prupinsi kalimantan barat indunisia géo...`
-3. `sabuah kampung di kacamatan solokuro kabupatin lamongan prupinsi jawa timur indunisia jujuhutan`
+1. `kampung di kacamatan tambakrejo kabupatin bojonegoro prupinsi jawa timur jujuhutan`
+2. `adalah sabuah kampung di kacamatan semitau kabupatin kapuas hulu prupinsi kalimantan barat indunisia...`
+3. `sabuah kampung di kacamatan long iram kabupatin kutai barat prupinsi kalimantan timur indunisia géog...`
 
 **Context Size 4:**
 
-1. `sabuah kampung di kacamatan muara badak kabupatin kutai kartanegara prupinsi kalimantan timur induni...`
-2. `adalah sabuah kampung di kacamatan kota bangun kabupatin kutai kartanegara prupinsi kalimantan timur...`
-3. `sabuting kampung di kacamatan pulau laut timur kabupatin kotabaru prupinsi kalimantan selatan induni...`
+1. `sabuah kampung di kacamatan bengalon kabupatin kutai timur prupinsi kalimantan timur indunisia indun...`
+2. `adalah sabuah kampung di kacamatan ketungau tengah kabupatin sintang prupinsi kalimantan barat indun...`
+3. `yaitu sabuting kampung di kacamatan karang intan kabupatin banjar prupinsi kalimantan selatan induni...`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `ah_beratelancit_`
-2. `_k_harangipin_pa`
-3. `n_seajudan_ancar`
+1. `awanaangik_ta,_t`
+2. `_g_viabara_pa_li`
+3. `ng_ksawarbunteru`
 
 **Context Size 2:**
 
-1. `antar_hunimungara`
-2. `n_vilet,_dimbuan_`
-3. `ng._ikuah_nanti_h`
+1. `anyan_adangga,_br`
+2. `n_kalambang_pem_a`
+3. `ng_dew,_dibantu,_`
 
 **Context Size 3:**
 
-1. `an_hijau_duwa_dino`
-2. `angai_wataceh_tan_`
-3. `_kamiri'iai_(di_ka`
+1. `an_jejani_andan_ka`
+2. `ang_sambara,_pres,`
+3. `_kacamatas_palima_`
 
 **Context Size 4:**
 
-1. `ang_paningkayat_nan`
-2. `tan_sabalumnya,_jum`
-3. `nang_labu_adalah_ra`
+1. `ang_maman_banjadi_h`
+2. `tan_bakcanganis_rik`
+3. `nang_kampung_dalah_`
 
 
 ### Key Findings
 
 - **Best Predictability:** Context-4 (word) with 98.5% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (179,116 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (179,736 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,48 +346,48 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 41,097 |
-| Total Tokens | 980,207 |
-| Mean Frequency | 23.85 |
+| Vocabulary Size | 41,351 |
+| Total Tokens | 992,449 |
+| Mean Frequency | 24.00 |
 | Median Frequency | 4 |
-| Frequency Std Dev | 276.25 |
+| Frequency Std Dev | 278.70 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | di | 27,381 |
-| 2 | nang | 26,981 |
-| 3 | wan | 16,949 |
-| 4 | adalah | 10,740 |
-| 5 | indunisia | 9,397 |
-| 6 | lawan | 9,395 |
-| 7 | kacamatan | 9,110 |
-| 8 | kalimantan | 8,310 |
-| 9 | kampung | 7,769 |
-| 10 | matan | 7,559 |
+| 1 | di | 27,655 |
+| 2 | nang | 27,387 |
+| 3 | wan | 17,250 |
+| 4 | adalah | 10,715 |
+| 5 | lawan | 9,581 |
+| 6 | indunisia | 9,420 |
+| 7 | kacamatan | 9,139 |
+| 8 | kalimantan | 8,368 |
+| 9 | kampung | 7,824 |
+| 10 | matan | 7,698 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | subreddit | 2 |
-| 2 | falco | 2 |
-| 3 | altarnatif | 2 |
-| 4 | elevasi | 2 |
-| 5 | kapus | 2 |
-| 6 | klayangan | 2 |
-| 7 | simbangan | 2 |
-| 8 | basyah | 2 |
-| 9 | simanggu | 2 |
-| 10 | kaliningan | 2 |
+| 1 | beregszásziová | 2 |
+| 2 | košice | 2 |
+| 3 | satian | 2 |
+| 4 | extreme | 2 |
+| 5 | frisna | 2 |
+| 6 | ropang | 2 |
+| 7 | caknan | 2 |
+| 8 | muktamar | 2 |
+| 9 | sandon | 2 |
+| 10 | sékuéns | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 1.0489 |
-| R² (Goodness of Fit) | 0.995015 |
+| Zipf Coefficient | 1.0491 |
+| R² (Goodness of Fit) | 0.995109 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
@@ -363,15 +395,15 @@ Below are text samples generated from each subword-based Markov chain model:
 | Top N Words | Coverage |
 |-------------|----------|
 | Top 100 | 35.5% |
-| Top 1,000 | 62.3% |
+| Top 1,000 | 62.4% |
 | Top 5,000 | 81.6% |
 | Top 10,000 | 88.8% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9950 indicates excellent adherence to Zipf's law
+- **Zipf Compliance:** R²=0.9951 indicates excellent adherence to Zipf's law
 - **High Frequency Dominance:** Top 100 words cover 35.5% of corpus
-- **Long Tail:** 31,097 words needed for remaining 11.2% coverage
+- **Long Tail:** 31,351 words needed for remaining 11.2% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.8673 🏆 | 0.3425 | N/A | N/A |
-| **mono_64d** | 64 | 0.8381 | 0.2559 | N/A | N/A |
-| **mono_128d** | 128 | 0.5573 | 0.2314 | N/A | N/A |
+| **mono_32d** | 32 | 0.8715 | 0.3303 | N/A | N/A |
+| **mono_64d** | 64 | 0.8409 | 0.2593 | N/A | N/A |
+| **mono_128d** | 128 | 0.5527 | 0.2130 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8715 🏆 | 0.3312 | 0.0420 | 0.2520 |
+| **aligned_64d** | 64 | 0.8409 | 0.2582 | 0.0680 | 0.3160 |
+| **aligned_128d** | 128 | 0.5527 | 0.2256 | 0.1380 | 0.4260 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.8673 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.2766. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** aligned_32d with 0.8715 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.2696. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 13.8% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **0.423** | High formulaic/idiomatic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -426,26 +461,26 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Prefixes
 | Prefix | Examples |
 |--------|----------|
-| `-ma` | malayisasi, maajuakan, mamake |
-| `-pa` | padatuannya, parlumbaan, panyalidikan |
-| `-di` | dirasmikan, ditarima, diversity |
-| `-ba` | babanyu, bacalak, badatang |
-| `-ka` | karaktir, kavi, kailmuan |
-| `-me` | melimpah, mesoamerika, menceritakan |
-| `-man` | manuntun, manguap, mangurbanakan |
-| `-pe` | perhubungan, perpaduan, peraih |
+| `-ma` | manentang, maut, marked |
+| `-pa` | parachute, pattern, pamain |
+| `-ba` | bantam, babakan, barambai |
+| `-di` | dibawakan, dihimpun, dibatasi |
+| `-ka` | karoseri, kampanye, kahala |
+| `-ta` | tatikap, tahitung, tato |
+| `-man` | manentang, manuruti, manggalungsur |
+| `-pe` | penyelenggara, pengadilan, pertapaan |
 
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-n` | perhubungan, dirasmikan, parlumbaan |
-| `-an` | perhubungan, dirasmikan, parlumbaan |
-| `-a` | padatuannya, ditarima, arménia |
-| `-ng` | sekapung, jelutung, badatang |
-| `-ya` | padatuannya, syairnya, sautingnya |
-| `-nya` | padatuannya, syairnya, sautingnya |
-| `-kan` | dirasmikan, maajuakan, panyalidikan |
-| `-akan` | maajuakan, menceritakan, mambedakan |
+| `-n` | pattern, babakan, tikinan |
+| `-an` | babakan, tikinan, kanaan |
+| `-a` | kurbannya, kahala, dhaka |
+| `-ng` | manentang, gondang, rahang |
+| `-kan` | babakan, dibawakan, menguntungkan |
+| `-ya` | kurbannya, karibnya, makanannya |
+| `-nya` | kurbannya, karibnya, makanannya |
+| `-akan` | babakan, dibawakan, maruntuhakan |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -453,18 +488,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `anga` | 1.72x | 223 contexts | tanga, sanga, angah |
-| `unga` | 1.95x | 57 contexts | bunga, tungau, sungay |
-| `ntan` | 1.94x | 48 contexts | antan, intan, rentan |
-| `ngan` | 1.85x | 58 contexts | bongan, ringan, mangan |
-| `anja` | 1.68x | 82 contexts | ganja, anjat, sanja |
-| `ting` | 1.59x | 79 contexts | piting, tingah, eating |
-| `ndun` | 2.11x | 23 contexts | rundun, indung, bendung |
-| `mant` | 1.80x | 39 contexts | manta, manti, mantah |
-| `dala` | 1.69x | 38 contexts | dalas, dalam, adalah |
-| `pung` | 1.88x | 21 contexts | apung, tapung, oppung |
-| `atin` | 1.75x | 26 contexts | patin, satin, atina |
-| `mpun` | 1.61x | 27 contexts | ampun, impun, rumpun |
+| `anga` | 1.62x | 225 contexts | sanga, manga, nanga |
+| `unga` | 2.11x | 57 contexts | bunga, rungan, bungas |
+| `ngan` | 1.95x | 58 contexts | pangan, rungan, bongan |
+| `anja` | 1.76x | 82 contexts | sanja, ganja, anjat |
+| `ntan` | 1.89x | 49 contexts | antan, intan, antang |
+| `mant` | 1.94x | 39 contexts | manta, manti, mantel |
+| `ting` | 1.63x | 79 contexts | keting, tingah, eating |
+| `ndun` | 2.15x | 24 contexts | rundun, indung, mendung |
+| `dala` | 1.77x | 38 contexts | dalam, dalas, adalah |
+| `atin` | 1.82x | 26 contexts | atina, batin, latin |
+| `pung` | 1.91x | 21 contexts | apung, pungsi, capung |
+| `adal` | 1.91x | 16 contexts | badal, kadal, adalah |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -472,16 +507,16 @@ This table shows which prefixes and suffixes most frequently co-occur on the sam
 
 | Prefix | Suffix | Frequency | Examples |
 |--------|--------|-----------|----------|
-| `-pa` | `-n` | 239 words | panulisan, panjagaan |
-| `-pa` | `-an` | 228 words | panulisan, panjagaan |
-| `-di` | `-n` | 165 words | dibandingakan, didasarakan |
-| `-di` | `-an` | 159 words | dibandingakan, didasarakan |
-| `-ma` | `-n` | 156 words | manurunakan, manyaurangan |
-| `-di` | `-kan` | 155 words | dibandingakan, didasarakan |
-| `-ma` | `-an` | 145 words | manurunakan, manyaurangan |
-| `-ka` | `-n` | 144 words | kaharuddin, kawarganegaraan |
-| `-ka` | `-an` | 138 words | kawarganegaraan, katabalan |
-| `-ma` | `-kan` | 127 words | manurunakan, mahamburakan |
+| `-pa` | `-n` | 207 words | palayanan, paampihan |
+| `-pa` | `-an` | 195 words | palayanan, paampihan |
+| `-di` | `-n` | 149 words | diasingakan, dimanangakan |
+| `-ma` | `-n` | 144 words | manyurangan, mampartahanakan |
+| `-ka` | `-n` | 144 words | kamantirian, kajiwaan |
+| `-di` | `-an` | 140 words | diasingakan, dimanangakan |
+| `-ma` | `-an` | 136 words | manyurangan, mampartahanakan |
+| `-di` | `-kan` | 133 words | diasingakan, dimanangakan |
+| `-ka` | `-an` | 133 words | kamantirian, kajiwaan |
+| `-ma` | `-kan` | 126 words | mampartahanakan, maungkaiakan |
 
 ### 6.5 Recursive Morpheme Segmentation
 
@@ -489,26 +524,28 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
-| kadatangannya | **`ka-data-ng-an-nya`** | 9.0 | `data` |
-| dikaluarkannya | **`di-ka-luar-kan-nya`** | 9.0 | `luar` |
-| badatangan | **`ba-data-ng-an`** | 7.5 | `data` |
-| dibayangakan | **`di-ba-yang-akan`** | 7.5 | `yang` |
-| kahabangan | **`ka-haba-ng-an`** | 7.5 | `haba` |
-| ditinggalakannya | **`di-tinggal-akan-nya`** | 7.5 | `tinggal` |
-| kakuasaannya | **`ka-kuasa-an-nya`** | 7.5 | `kuasa` |
-| didinginakan | **`di-di-ngin-akan`** | 7.5 | `ngin` |
-| mamakainya | **`ma-ma-ka-inya`** | 7.5 | `inya` |
-| kaakhirannya | **`ka-akhir-an-nya`** | 7.5 | `akhir` |
-| kadalamannya | **`ka-dalam-an-nya`** | 7.5 | `dalam` |
-| katakutanan | **`ka-takut-an-an`** | 7.5 | `takut` |
-| kanyataannya | **`ka-nyata-an-nya`** | 7.5 | `nyata` |
-| dikeluarakan | **`di-keluar-akan`** | 6.0 | `keluar` |
-| batanaman | **`ba-tanam-an`** | 6.0 | `tanam` |
+| kaputingannya | **`ka-puti-ng-an-nya`** | 9.0 | `puti` |
+| dimanpaatakan | **`di-man-pa-atak-an`** | 9.0 | `atak` |
+| manjadiakannya | **`man-jadi-akan-nya`** | 7.5 | `jadi` |
+| mamandiakan | **`ma-man-di-akan`** | 7.5 | `akan` |
+| disayangakan | **`di-sa-yang-akan`** | 7.5 | `yang` |
+| peradangan | **`pe-rada-ng-an`** | 7.5 | `rada` |
+| dimakamakan | **`di-ma-ka-makan`** | 7.5 | `makan` |
+| kakacangan | **`ka-ka-cang-an`** | 7.5 | `cang` |
+| disalanggarakan | **`di-sa-langgar-akan`** | 7.5 | `langgar` |
+| takapinggirakan | **`ta-ka-pinggir-akan`** | 7.5 | `pinggir` |
+| dihasilakannya | **`di-hasil-akan-nya`** | 7.5 | `hasil` |
+| pahitungan | **`pa-hitu-ng-an`** | 7.5 | `hitu` |
+| papadahannya | **`pa-pa-dahan-nya`** | 7.5 | `dahan` |
+| sabalumannya | **`sa-ba-luman-nya`** | 7.5 | `luman` |
+| kahiringan | **`ka-hiri-ng-an`** | 7.5 | `hiri` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language BJN appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Banjar shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
+
+> **Note on Idiomaticity:** The high Idiomaticity Gap suggests a large number of frequent multi-word expressions or formulaic sequences that are statistically distinct from their component parts.
 
 ---
 ## 7. Summary & Recommendations
@@ -520,7 +557,7 @@ The language BJN appears to be more isolating or has a highly fixed vocabulary.
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
 | Tokenizer | **64k BPE** | Best compression (4.83x) |
-| N-gram | **2-gram** | Lowest perplexity (186) |
+| N-gram | **2-gram** | Lowest perplexity (185) |
 | Markov | **Context-4** | Highest predictability (98.5%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
@@ -735,4 +772,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 07:24:26*
+*Report Date: 2026-01-03 19:11:59*

models/embeddings/aligned/bjn_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5b1376fe24e9ed09f7a37f65ebcc26d7775683af2dce4a0ddc201ba717092d7
+size 1044955202

models/embeddings/aligned/bjn_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "bjn", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bjn_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7659e433a7af24229e0c3f240e4d25dea875e1f239b3e23e2966e850fb16dec5
+size 65664

models/embeddings/aligned/bjn_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bjn",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 5941,
+  "vocab_size": 20133
+}

models/embeddings/aligned/bjn_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bc1da0f0899c29e0ef01c58e6fb37639ed96e4a58a24472ac1661dcdd5e356c
+size 261493058

models/embeddings/aligned/bjn_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "bjn", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bjn_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cca5e2396381002102d82a1f0af3b058de4c2e6a3eecb5fb9da7b85cc187c1e5
+size 4224

models/embeddings/aligned/bjn_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bjn",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 5941,
+  "vocab_size": 20133
+}

models/embeddings/aligned/bjn_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d8508862461e2b3899fb13f6a789921fd85a1f73b71aa0982f9456ee258c5aa
+size 522647106

models/embeddings/aligned/bjn_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "bjn", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bjn_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a77b0ff4e2ce9a25817bcb42b64f48afec46c70ebbf0b01bdc689a81c2cdcbc1
+size 16512

models/embeddings/aligned/bjn_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bjn",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 5941,
+  "vocab_size": 20133
+}

models/embeddings/monolingual/bjn_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e4e093c6a31e77a391c65fb05175223839736144a137894bb279acedb38a739e
-size 1044759476
+oid sha256:c5b1376fe24e9ed09f7a37f65ebcc26d7775683af2dce4a0ddc201ba717092d7
+size 1044955202

models/embeddings/monolingual/bjn_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 19945
+  "vocab_size": 20133
 }

models/embeddings/monolingual/bjn_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5dc1832ebecd5f5dafd473e3651dd39adeeb3b7babccfe0f41a0ff07bead9833
-size 261441716
+oid sha256:4bc1da0f0899c29e0ef01c58e6fb37639ed96e4a58a24472ac1661dcdd5e356c
+size 261493058

models/embeddings/monolingual/bjn_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 19945
+  "vocab_size": 20133
 }

models/embeddings/monolingual/bjn_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:01dad40fd9ad207e7764f6c7896ea76ca31b58aac8fc24673d9cc7aaf6e7e1dd
-size 522547636
+oid sha256:5d8508862461e2b3899fb13f6a789921fd85a1f73b71aa0982f9456ee258c5aa
+size 522647106

models/embeddings/monolingual/bjn_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 19945
+  "vocab_size": 20133
 }

models/subword_markov/bjn_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7ada7af32fa948a0655b7ebf55bc43135b6b1ac0a6182109dbe7b9ab245e5697
-size 91151
+oid sha256:d0075b36c13716e538d66bc6c95b0a7098dea9e5d3afadeb2bf746f3bccbc77e
+size 93718

models/subword_markov/bjn_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "bjn",
-  "unique_contexts": 2413,
-  "total_transitions": 7333167
+  "unique_contexts": 2416,
+  "total_transitions": 7420228
 }

models/subword_markov/bjn_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad84eb896f07adf50c1c8330a4fd412521e79c8611f16dbd54db03976d031b2a
-size 394385
+oid sha256:c06e01c61018a6528ae1dfe9272e8eab428d4cf9d68b20082caa08a8e0394e38
+size 394917

models/subword_markov/bjn_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "bjn",
-  "unique_contexts": 11071,
-  "total_transitions": 7321716
+  "unique_contexts": 11092,
+  "total_transitions": 7408725
 }

models/subword_markov/bjn_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:629f8e2533e8eb22ae3400a41bd35f7de6abee27c253d69864fba53bf91a7cab
-size 1413172
+oid sha256:5be9329a6e42ce8e7fd4e60a640ac7952161fffb037b322466de084f418cb822
+size 1413600

models/subword_markov/bjn_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "bjn",
-  "unique_contexts": 46029,
-  "total_transitions": 7310265
+  "unique_contexts": 46118,
+  "total_transitions": 7397222
 }

models/subword_markov/bjn_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f91a26206e37ee242d2c77719cfea85fb5572aabe4780ad4ddc16f7661bef71
-size 3779369
+oid sha256:e926fa10beb5ecf82ce531d290a8774002e61a0e21713340dd9050e7425f54b2
+size 3798519

models/subword_markov/bjn_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "bjn",
-  "unique_contexts": 179116,
-  "total_transitions": 7298814
+  "unique_contexts": 179736,
+  "total_transitions": 7385719
 }

models/subword_ngram/bjn_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f14b3c618cdbe3ba5b783d54065bf7f34a9d7f1b4144d0953086f77baae8376
-size 37041
+oid sha256:2ae9f98459248ee75d889469f9fbf164f87d751cd75c085207e3c5291b807d8f
+size 37388

models/subword_ngram/bjn_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "bjn",
-  "unique_ngrams": 2773,
-  "total_ngrams": 7333167
+  "unique_ngrams": 2788,
+  "total_ngrams": 7420228
 }

models/subword_ngram/bjn_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:353eff0669e301d0a373b59a8360bbc5dca472bf6870e145137cb0b3ea1388df
-size 245202
+oid sha256:b89eefc90605d6a52225b684fc14e91fcc90caf7a5d2a4e8cc64d811c7346de3
+size 246471

models/subword_ngram/bjn_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "bjn",
-  "unique_ngrams": 20264,
-  "total_ngrams": 7321716
+  "unique_ngrams": 20293,
+  "total_ngrams": 7408725
 }

models/subword_ngram/bjn_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4f49217b5541a90584ec8de80222bf07caa7874767428ec91809828776b503d
-size 1113155
+oid sha256:02479aa7f3aafd426205e27017151d37389a25dfa9cd3c47d323ec88b521bdf4
+size 1118422

models/subword_ngram/bjn_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "bjn",
-  "unique_ngrams": 99301,
-  "total_ngrams": 7310265
+  "unique_ngrams": 99642,
+  "total_ngrams": 7397222
 }

models/subword_ngram/bjn_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17b7616fd5a5e86d548f6b121644238bf3a02dd74a9b59b4ec28dcbfde1a8055
+size 2772560

models/subword_ngram/bjn_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "bjn",
+  "unique_ngrams": 245459,
+  "total_ngrams": 7385719
+}

models/tokenizer/bjn_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c3f5b630291acf2ba47ddad779b9b7da931da16ba83757be5051cc96de210969
-size 506686
+oid sha256:faaddd93773e7c2ec9a099b8cc18f32a74875884c023c10f7dd41fc2ca607f5e
+size 507025

models/tokenizer/bjn_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:909287240aaef4cf222642164f43a915d96db473274547f67f0fad9b37c325b0
-size 783745
+oid sha256:8873930010b0bebd5f37d1b88ab7c10613f1b408839e2c9eec517272abc4bec0
+size 783254

models/tokenizer/bjn_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b486f1ebd0107b543a8cc67ef5b38a3e84bc04e29bc7c4a065808b100089559d
-size 1358958
+oid sha256:8e0d5c97befc71a1a981d127b0e6d1c65b7ad49d3429cb2ebe20a69c1650dabf
+size 1359311

models/tokenizer/bjn_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:12bb0d65ad2471af55c560a13bb81daac89ac54df022206918fc85d8bd8cbf7a
-size 372387
+oid sha256:c3440f1d40cdbea3958bdf2966d52e30a39ccec52dbf325ac694beee20a30016
+size 372355

models/vocabulary/bjn_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:89e712e174d699814d715ee756db596976724acb27e04ba9b879b084c92b678b
-size 697891
+oid sha256:86d73a4fd441f201de90ea8dea3e376477144eb9a2b7a08ede3054e833c7a72c
+size 702399

models/vocabulary/bjn_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "bjn",
-  "vocabulary_size": 41097,
+  "vocabulary_size": 41351,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.09502040061134379,
+    "type_token_ratio": 0.09448990569765173,
     "coverage": {
-      "top_100": 0.3351626748795911,
-      "top_1000": 0.5889079264614335,
-      "top_5000": 0.7705822606228692,
-      "top_10000": 0.8384362165297953
+      "top_100": 0.3351965306637343,
+      "top_1000": 0.5892939938782019,
+      "top_5000": 0.7708286324969414,
+      "top_10000": 0.8386282602383026
     },
-    "hapax_count": 57507,
-    "hapax_ratio": 0.5832116344164537,
-    "total_documents": 11451
+    "hapax_count": 57896,
+    "hapax_ratio": 0.5833526454200127,
+    "total_documents": 11503
   }
 }

models/word_markov/bjn_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be08d60f4e38caedbe81491c069935efc94c9f044fae86f70e1ce4a81d7d2269
-size 3894998
+oid sha256:c30af757d17048fd3365266d6ec2b832b58a7202c43123769620aacbc8ed1351
+size 3918586

models/word_markov/bjn_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "bjn",
-  "unique_contexts": 98412,
-  "total_transitions": 1026263
+  "unique_contexts": 99056,
+  "total_transitions": 1038842
 }

models/word_markov/bjn_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b38f524f3d4a7fc742e5e51668a0ad8b6f77409c9a3d88cabe8baeca1b75ce5
-size 9782557
+oid sha256:a236fb6d637e19c7de9cbe0dac3c2574566b57ac6b6cceaeafd9484c73be2483
+size 9916222

models/word_markov/bjn_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "bjn",
-  "unique_contexts": 553823,
-  "total_transitions": 1014812
+  "unique_contexts": 559810,
+  "total_transitions": 1027339
 }