Upload all models and assets for bew (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: bew
-language_name: BEW
+language_name: Betawi
 language_family: austronesian_malay
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-austronesian_malay
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.487
+    value: 4.630
   - name: best_isotropy
     type: isotropy
-    value: 0.7802
+    value: 0.7504
   - name: vocabulary_size
     type: vocab
-    value: 18978
-generated: 2025-12-28
+    value: 0
+generated: 2026-01-03
 ---
 
-# BEW - Wikilangs Models
+# Betawi - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **BEW** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Betawi** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -44,12 +54,13 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 ### Models & Assets
 
 - Tokenizers (8k, 16k, 32k, 64k)
-- N-gram models (2, 3, 4-gram)
-- Markov chains (context of 1, 2, 3 and 4)
+- N-gram models (2, 3, 4, 5-gram)
+- Markov chains (context of 1, 2, 3, 4 and 5)
 - Subword N-gram and Markov chains
-- Embeddings in various sizes and dimensions
+- Embeddings in various sizes and dimensions (aligned and unaligned)
 - Language Vocabulary
 - Language Statistics
+
 ![Performance Dashboard](visualizations/performance_dashboard.png)
 
 ### Analysis and Evaluation
@@ -59,7 +70,8 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Summary & Recommendations](#6-summary--recommendations)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
+- [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
 
@@ -68,51 +80,57 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 ![Tokenizer Compression](visualizations/tokenizer_compression.png)
 
+![Tokenizer Fertility](visualizations/tokenizer_fertility.png)
+
+![Tokenizer OOV](visualizations/tokenizer_oov.png)
+
+![Total Tokens](visualizations/tokenizer_total_tokens.png)
+
 ### Results
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.702x | 3.68 | 0.1337% | 174,282 |
-| **16k** | 3.991x | 3.96 | 0.1441% | 161,647 |
-| **32k** | 4.245x | 4.22 | 0.1533% | 152,003 |
-| **64k** | 4.487x 🏆 | 4.46 | 0.1620% | 143,793 |
+| **8k** | 3.806x | 3.81 | 0.1398% | 155,259 |
+| **16k** | 4.118x | 4.13 | 0.1512% | 143,483 |
+| **32k** | 4.386x | 4.39 | 0.1611% | 134,715 |
+| **64k** | 4.630x 🏆 | 4.64 | 0.1700% | 127,635 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Pondok Melati entu kecamatan nyang ada di Bekasi Kota. Ni kecamatan ngejenggar a...`
+**Sample 1:** `D atawa hurup kecitnya d ya'entu hurup ke'ampat dalem hurup Latèn. Ruju'an Latèn`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁pondok ▁melati ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁bekasi ▁kota . ... (+18 more)` | 28 |
-| 16k | `▁pondok ▁melati ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁bekasi ▁kota . ... (+18 more)` | 28 |
-| 32k | `▁pondok ▁melati ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁bekasi ▁kota . ... (+18 more)` | 28 |
-| 64k | `▁pondok ▁melati ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁bekasi ▁kota . ... (+18 more)` | 28 |
+| 8k | `▁d ▁atawa ▁hurup ▁kecitnya ▁d ▁ya ' entu ▁hurup ▁ke ... (+11 more)` | 21 |
+| 16k | `▁d ▁atawa ▁hurup ▁kecitnya ▁d ▁ya ' entu ▁hurup ▁ke ... (+11 more)` | 21 |
+| 32k | `▁d ▁atawa ▁hurup ▁kecitnya ▁d ▁ya ' entu ▁hurup ▁ke ... (+10 more)` | 20 |
+| 64k | `▁d ▁atawa ▁hurup ▁kecitnya ▁d ▁ya ' entu ▁hurup ▁ke ... (+10 more)` | 20 |
 
-**Sample 2:** `Tanah Sarèal entu kecamatan nyang ada di Bogor Kota. Ni kecamatan ngejenggar amp...`
+**Sample 2:** `Karawaci entu kecamatan nyang ada di Tanggerang Kota. Ni kecamatan ngejenggar am...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁tanah ▁sar è al ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁bogor ... (+18 more)` | 28 |
-| 16k | `▁tanah ▁sar è al ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁bogor ... (+18 more)` | 28 |
-| 32k | `▁tanah ▁sarè al ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁bogor ▁kota ... (+17 more)` | 27 |
-| 64k | `▁tanah ▁sarèal ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁bogor ▁kota . ... (+16 more)` | 26 |
+| 8k | `▁kara wa ci ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁tanggerang ▁kota ... (+17 more)` | 27 |
+| 16k | `▁kara wa ci ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁tanggerang ▁kota ... (+17 more)` | 27 |
+| 32k | `▁kara wa ci ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁tanggerang ▁kota ... (+17 more)` | 27 |
+| 64k | `▁karawaci ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁tanggerang ▁kota . ▁ni ... (+15 more)` | 25 |
 
-**Sample 3:** `Forum lingkar pena ielah`
+**Sample 3:** `Limo entu kecamatan nyang ada di Dèpok Kota, Jawa Kulon, Indonésia. Ni kecamatan...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁forum ▁lingkar ▁pena ▁i elah` | 5 |
-| 16k | `▁forum ▁lingkar ▁pena ▁ielah` | 4 |
-| 32k | `▁forum ▁lingkar ▁pena ▁ielah` | 4 |
-| 64k | `▁forum ▁lingkar ▁pena ▁ielah` | 4 |
+| 8k | `▁li mo ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁dèpok ▁kota , ... (+21 more)` | 31 |
+| 16k | `▁limo ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁dèpok ▁kota , ▁jawa ... (+20 more)` | 30 |
+| 32k | `▁limo ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁dèpok ▁kota , ▁jawa ... (+20 more)` | 30 |
+| 64k | `▁limo ▁entu ▁kecamatan ▁nyang ▁ada ▁di ▁dèpok ▁kota , ▁jawa ... (+20 more)` | 30 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.487x compression
-- **Lowest UNK Rate:** 8k with 0.1337% unknown tokens
+- **Best Compression:** 64k achieves 4.630x compression
+- **Lowest UNK Rate:** 8k with 0.1398% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -121,57 +139,111 @@ Below are sample sentences tokenized with each vocabulary size:
 
 ![N-gram Perplexity](visualizations/ngram_perplexity.png)
 
+![N-gram Unique](visualizations/ngram_unique.png)
+
 ![N-gram Coverage](visualizations/ngram_coverage.png)
 
 ### Results
 
-| N-gram | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
-|--------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | 2,734 🏆 | 11.42 | 11,166 | 32.8% | 60.2% |
-| **2-gram** | 288 🏆 | 8.17 | 2,851 | 67.0% | 98.5% |
-| **3-gram** | 3,250 | 11.67 | 13,829 | 31.3% | 56.8% |
-| **3-gram** | 2,150 | 11.07 | 19,026 | 28.2% | 72.7% |
-| **4-gram** | 4,868 | 12.25 | 21,467 | 28.6% | 50.8% |
-| **4-gram** | 9,513 | 13.22 | 75,515 | 16.1% | 45.6% |
+| N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
+|--------|---------|------------|---------|----------------|------------------|-------------------|
+| **2-gram** | Word | 2,340 | 11.19 | 7,241 | 31.0% | 59.9% |
+| **2-gram** | Subword | 256 🏆 | 8.00 | 2,508 | 70.0% | 98.9% |
+| **3-gram** | Word | 1,985 | 10.95 | 6,755 | 33.8% | 62.9% |
+| **3-gram** | Subword | 1,910 | 10.90 | 16,523 | 30.0% | 74.7% |
+| **4-gram** | Word | 3,084 | 11.59 | 9,753 | 29.7% | 56.5% |
+| **4-gram** | Subword | 8,721 | 13.09 | 66,990 | 16.6% | 46.5% |
+| **5-gram** | Word | 1,919 | 10.91 | 5,996 | 33.6% | 65.3% |
+| **5-gram** | Subword | 22,647 | 14.47 | 131,412 | 12.5% | 33.4% |
 
 ### Top 5 N-grams by Size
 
-**2-grams:**
+**2-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `arab gundul` | 3,312 |
+| 2 | `hurup arab` | 3,190 |
+| 3 | `ruju an` | 2,891 |
+| 4 | `ada di` | 1,396 |
+| 5 | `entu atu` | 1,364 |
+
+**3-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `hurup arab gundul` | 3,176 |
+| 2 | `nyang ada di` | 741 |
+| 3 | `ruju an di` | 723 |
+| 4 | `nyang tinggal di` | 641 |
+| 5 | `tinggal di mari` | 614 |
+
+**4-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `nyang tinggal di mari` | 609 |
+| 2 | `orang nyang tinggal di` | 600 |
+| 3 | `ruju an di indonésia` | 529 |
+| 4 | `nyang ada di propinsi` | 509 |
+| 5 | `km2 dengen kepadetan penduduknya` | 501 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `orang nyang tinggal di mari` | 584 |
+| 2 | `nyang tinggal di mari ruju` | 442 |
+| 3 | `tinggal di mari ruju an` | 442 |
+| 4 | `di mari ruju an di` | 440 |
+| 5 | `mari ruju an di indonésia` | 438 |
+
+**2-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `a n` | 74,827 |
+| 2 | `a _` | 60,507 |
+| 3 | `n g` | 54,383 |
+| 4 | `n _` | 46,937 |
+| 5 | `_ a` | 35,570 |
+
+**3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `gundul :` | 3,517 |
-| 2 | `category :` | 3,400 |
-| 3 | `( hurup` | 3,387 |
-| 4 | `arab gundul` | 3,312 |
-| 5 | `hurup arab` | 3,190 |
+| 1 | `n y a` | 27,185 |
+| 2 | `a n g` | 25,765 |
+| 3 | `n g _` | 25,518 |
+| 4 | `a n _` | 24,856 |
+| 5 | `_ d i` | 20,857 |
 
-**3-grams:**
+**4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `arab gundul :` | 3,306 |
-| 2 | `hurup arab gundul` | 3,176 |
-| 3 | `( hurup arab` | 3,132 |
-| 4 | `ruju ' an` | 2,848 |
-| 5 | `. ruju '` | 2,632 |
+| 1 | `a n g _` | 17,737 |
+| 2 | `n y a _` | 13,480 |
+| 3 | `_ d i _` | 10,268 |
+| 4 | `_ n y a` | 10,013 |
+| 5 | `y a n g` | 9,660 |
 
-**4-grams:**
+**5-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `hurup arab gundul :` | 3,171 |
-| 2 | `( hurup arab gundul` | 3,124 |
-| 3 | `. ruju ' an` | 2,631 |
-| 4 | `' an category :` | 1,377 |
-| 5 | `ruju ' an category` | 1,375 |
+| 1 | `y a n g _` | 9,531 |
+| 2 | `_ n y a n` | 9,175 |
+| 3 | `n y a n g` | 9,145 |
+| 4 | `_ a m a _` | 5,520 |
+| 5 | `e n t u _` | 5,202 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram with 288
+- **Best Perplexity:** 2-gram (subword) with 256
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~46% of corpus
+- **Coverage:** Top-1000 patterns cover ~33% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -179,55 +251,86 @@ Below are sample sentences tokenized with each vocabulary size:
 
 ![Markov Entropy](visualizations/markov_entropy.png)
 
+![Markov Contexts](visualizations/markov_contexts.png)
+
 ![Markov Branching](visualizations/markov_branching.png)
 
 ### Results
 
-| Context | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
-|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | 0.7488 | 1.680 | 4.84 | 43,321 | 25.1% |
-| **1** | 0.9960 | 1.994 | 6.07 | 1,469 | 0.4% |
-| **2** | 0.2696 | 1.205 | 1.61 | 209,140 | 73.0% |
-| **2** | 0.8519 | 1.805 | 4.62 | 8,912 | 14.8% |
-| **3** | 0.0871 | 1.062 | 1.16 | 336,473 | 91.3% |
-| **3** | 0.7791 | 1.716 | 3.44 | 41,145 | 22.1% |
-| **4** | 0.0352 🏆 | 1.025 | 1.07 | 389,729 | 96.5% |
-| **4** | 0.5441 🏆 | 1.458 | 2.28 | 141,337 | 45.6% |
+| Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
+|---------|---------|-------------|------------|------------------|-----------------|----------------|
+| **1** | Word | 0.8296 | 1.777 | 4.87 | 41,205 | 17.0% |
+| **1** | Subword | 0.7866 | 1.725 | 4.95 | 1,639 | 21.3% |
+| **2** | Word | 0.2134 | 1.159 | 1.43 | 200,219 | 78.7% |
+| **2** | Subword | 0.7991 | 1.740 | 4.40 | 8,105 | 20.1% |
+| **3** | Word | 0.0565 | 1.040 | 1.10 | 285,266 | 94.3% |
+| **3** | Subword | 0.7622 | 1.696 | 3.43 | 35,638 | 23.8% |
+| **4** | Word | 0.0212 🏆 | 1.015 | 1.04 | 311,018 | 97.9% |
+| **4** | Subword | 0.5570 | 1.471 | 2.34 | 122,163 | 44.3% |
+
+### Generated Text Samples (Word-based)
+
+Below are text samples generated from each word-based Markov chain model:
+
+**Context Size 1:**
+
+1. `di mari per sènsus tahon wayah ada singa laut malah bulu tepok tepok bulu dènemarken juga`
+2. `nyang gocan berobah beneran gim kumpiuter hal ada 412 ama jadi dedengkot soldadu romèn hurup arap`
+3. `ama kemajuan èkonomi kecil bakal dipisahin deri prasman tchad arab gundul ايسيت ièlah orang nyang ad...`
+
+**Context Size 2:**
+
+1. `arab gundul سورين entu tana rumput rata ada banyak bodoran tasawup nyang kenisbat ke dia punya anggu...`
+2. `hurup arab gundul دمفا indonésia herpes nyang pires dampa ringkes hsv ièlah atu bangunan dasaran nya...`
+3. `ruju an enclekan wikimédia jakarta`
+
+**Context Size 3:**
+
+1. `hurup arab gundul عصر atawa sembayang asar hurup arab gundul فراولين di kaèdah basa entu penglakon d...`
+2. `nyang ada di propinsi jawa tenga ni kabupatèn punya sintrem guwernemèn ada di jailolo ni kabupatèn n...`
+3. `ruju an di indonésia tenga kota`
+
+**Context Size 4:**
+
+1. `nyang tinggal di mari di indonésia tenga`
+2. `orang nyang tinggal di mari ruju an di indonésia kulon kota`
+3. `nyang ada di propinsi jawa tenga ni kabupatèn punya sintrem guwernemèn ada di pati ni kabupatèn ngej...`
 
-### Generated Text Samples
 
-Below are text samples generated from each Markov chain model:
+### Generated Text Samples (Subword-based)
+
+Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `. barung nyang tinggal di awalnya ga tau - laèn - terusan . 15 agustus 1945`
-2. `, jadinya orang / 347 sm , canada . ada punya waktu junior , sirilik :`
-3. `: rectum , 3 taon ( uè ] ˤəʔ ( 2005 ) ièlah pahlawan nasional ,`
+1. `_t,_naèsa_(in_an`
+2. `ah_ha_n_psc_sèn,`
+3. `nalianyanngele-d`
 
 **Context Size 2:**
 
-1. `gundul : ارسخ ; basa katalan : espanya ; basa pin ama swèd , cuman enni kué`
-2. `category : papua nugini category : kota di bilangan èropa kidul - wètan . ni basa bunyi`
-3. `( hurup arab gundul : کبليک ) – ibukota ( indo . ) , atawa nyang seantéro`
+1. `andanésin_nya_at.`
+2. `a_dongan_1_jen._d`
+3. `ng_bensia_or._ret`
 
 **Context Size 3:**
 
-1. `arab gundul : ارلوجى کنتوڠ ) atawa horloji kantong ( hurup arab gundul : تورون بروء ) ièlah`
-2. `hurup arab gundul : موبل بک ) atawa losbak ( hurup arab gundul : خرکف ; rus :`
-3. `( hurup arab gundul : غزة ; arab : تشاد tsyād ) atawa resminya kiblik nigèr ( hurup`
+1. `nya_ke_1:_6_ada_de`
+2. `ang))_atu_kulon_de`
+3. `ng_nya_punya,_kota`
 
 **Context Size 4:**
 
-1. `hurup arab gundul : چلان ) entu pakéan luaran nyang nutupin pinggang ampé kekiongan , kadang cuman e...`
-2. `( hurup arab gundul : ) atawa gedong lèr pèjèngan ( hurup arab gundul : ) entu kejadian cuaca`
-3. `. ruju ' an sènggètan luar martunis sarbini di instagram martunis ronaldo di instagram martunis rona...`
+1. `ang_damé_kalannya_b`
+2. `nya_design:top;padd`
+3. `_di_kota_lingking_k`
 
 
 ### Key Findings
 
-- **Best Predictability:** Context-4 with 96.5% predictability
+- **Best Predictability:** Context-4 (word) with 97.9% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (141,337 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (122,163 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -243,64 +346,64 @@ Below are text samples generated from each Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 18,978 |
-| Total Tokens | 362,916 |
-| Mean Frequency | 19.12 |
+| Vocabulary Size | 18,200 |
+| Total Tokens | 340,971 |
+| Mean Frequency | 18.73 |
 | Median Frequency | 4 |
-| Frequency Std Dev | 165.48 |
+| Frequency Std Dev | 164.32 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | di | 10,336 |
-| 2 | nyang | 9,101 |
+| 1 | di | 10,322 |
+| 2 | nyang | 9,100 |
 | 3 | ama | 5,533 |
-| 4 | entu | 5,338 |
-| 5 | ada | 4,150 |
-| 6 | atawa | 3,974 |
+| 4 | entu | 5,337 |
+| 5 | ada | 4,148 |
+| 6 | atawa | 3,973 |
 | 7 | ni | 3,950 |
-| 8 | orang | 3,897 |
-| 9 | punya | 3,836 |
-| 10 | hurup | 3,665 |
+| 8 | punya | 3,836 |
+| 9 | hurup | 3,638 |
+| 10 | arab | 3,568 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | abi | 2 |
-| 2 | gelanggang | 2 |
-| 3 | m² | 2 |
-| 4 | writing | 2 |
-| 5 | syaamil | 2 |
-| 6 | fermentasi | 2 |
-| 7 | xl | 2 |
+| 1 | kirinya | 2 |
+| 2 | ngeloncat | 2 |
+| 3 | abi | 2 |
+| 4 | gelanggang | 2 |
+| 5 | writing | 2 |
+| 6 | syaamil | 2 |
+| 7 | fermentasi | 2 |
 | 8 | oase | 2 |
-| 9 | sos | 2 |
-| 10 | litt | 2 |
+| 9 | maimon | 2 |
+| 10 | herawati | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 1.0759 |
-| R² (Goodness of Fit) | 0.994882 |
+| Zipf Coefficient | 1.0754 |
+| R² (Goodness of Fit) | 0.994702 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 41.4% |
-| Top 1,000 | 69.4% |
-| Top 5,000 | 87.6% |
-| Top 10,000 | 94.3% |
+| Top 100 | 41.8% |
+| Top 1,000 | 69.7% |
+| Top 5,000 | 87.8% |
+| Top 10,000 | 94.6% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9949 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 41.4% of corpus
-- **Long Tail:** 8,978 words needed for remaining 5.7% coverage
+- **Zipf Compliance:** R²=0.9947 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 41.8% of corpus
+- **Long Tail:** 8,200 words needed for remaining 5.4% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -313,24 +416,135 @@ Below are text samples generated from each Markov chain model:
 
 ![t-SNE Sentences](visualizations/tsne_sentences.png)
 
-### Model Comparison
 
-| Model | Vocab Size | Dimension | Avg Norm | Std Norm | Isotropy |
-|-------|------------|-----------|----------|----------|----------|
-| **mono_32d** | 7,863 | 32 | 3.543 | 0.852 | 0.7802 🏆 |
-| **mono_64d** | 7,863 | 64 | 3.726 | 0.807 | 0.4496 |
-| **mono_128d** | 7,863 | 128 | 3.776 | 0.808 | 0.1049 |
-| **embeddings_enhanced** | 0 | 0 | 0.000 | 0.000 | 0.0000 |
+### 5.1 Cross-Lingual Alignment
+
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
+
+
+### 5.2 Model Comparison
+
+| Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
+|-------|-----------|----------|------------------|---------------|----------------|
+| **mono_32d** | 32 | 0.7504 | 0.3662 | N/A | N/A |
+| **mono_64d** | 64 | 0.4073 | 0.3304 | N/A | N/A |
+| **mono_128d** | 128 | 0.0951 | 0.3259 | N/A | N/A |
+| **aligned_32d** | 32 | 0.7504 🏆 | 0.3611 | 0.0280 | 0.1800 |
+| **aligned_64d** | 64 | 0.4073 | 0.3298 | 0.0640 | 0.2540 |
+| **aligned_128d** | 128 | 0.0951 | 0.3286 | 0.0840 | 0.2940 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.7802 (more uniform distribution)
-- **Dimension Trade-off:** Higher dimensions capture more semantics but reduce isotropy
-- **Vocabulary Coverage:** All models cover 7,863 words
-- **Recommendation:** 100d for balanced semantic capture and efficiency
+- **Best Isotropy:** aligned_32d with 0.7504 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.3404. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 8.4% R@1 in cross-lingual retrieval.
+- **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
-## 6. Summary & Recommendations
+## 6.  Morphological Analysis (Experimental)
+
+This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
+
+### 6.1 Productivity & Complexity
+
+| Metric | Value | Interpretation | Recommendation |
+|--------|-------|----------------|----------------|
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **0.957** | High formulaic/idiomatic content | - |
+
+### 6.2 Affix Inventory (Productive Units)
+
+These are the most productive prefixes and suffixes identified by sampling the vocabulary for global substitutability patterns. A unit is considered an affix if stripping it leaves a valid stem that appears in other contexts.
+
+#### Productive Prefixes
+| Prefix | Examples |
+|--------|----------|
+| `-pe` | perinta, pernahkan, pengablagan |
+| `-di` | dirangkèng, diplomat, dibelakonin |
+| `-ke` | kepri, kerbala, kesannya |
+| `-ng` | ngucap, ngelangsir, nglingkup |
+| `-se` | secret, sejarah, sexual |
+
+#### Productive Suffixes
+| Suffix | Examples |
+|--------|----------|
+| `-n` | pernahkan, pengablagan, waringin |
+| `-an` | pernahkan, pengablagan, tuan |
+| `-a` | perinta, kakinya, udara |
+| `-ya` | kakinya, bawaannya, kesannya |
+| `-nya` | kakinya, bawaannya, kesannya |
+| `-ng` | dirangkèng, peringgiorang, bambang |
+| `-in` | waringin, lanjutin, ngusahain |
+
+### 6.3 Bound Stems (Lexical Roots)
+
+Bound stems are high-frequency subword units that are semantically cohesive but rarely appear as standalone words. These often correspond to the 'core' of a word that requires inflection or derivation to be valid.
+
+| Stem | Cohesion | Substitutability | Examples |
+|------|----------|------------------|----------|
+| `anya` | 1.55x | 72 contexts | tanya, nanya, anyar |
+| `ngan` | 1.63x | 52 contexts | ongan, ringan, dengan |
+| `angg` | 1.48x | 64 contexts | kanggo, bangga, mangga |
+| `aran` | 1.38x | 71 contexts | maran, saran, garan |
+| `enga` | 1.61x | 36 contexts | senga, nenga, tenga |
+| `anny` | 1.68x | 27 contexts | annya, umannya, ujannya |
+| `unya` | 1.65x | 27 contexts | punya, baunya, atunya |
+| `rang` | 1.32x | 60 contexts | orang, prang, urang |
+| `inya` | 1.49x | 36 contexts | sinyal, minyak, arinya |
+| `atan` | 1.50x | 32 contexts | yatan, alatan, muatan |
+| `ling` | 1.41x | 40 contexts | aling, èling, beling |
+| `enge` | 1.48x | 25 contexts | pengen, tengen, denger |
+
+### 6.4 Affix Compatibility (Co-occurrence)
+
+This table shows which prefixes and suffixes most frequently co-occur on the same stems, revealing the 'stacking' rules of the language's morphology.
+
+| Prefix | Suffix | Frequency | Examples |
+|--------|--------|-----------|----------|
+| `-pe` | `-n` | 250 words | pengrongrongan, penyatetan |
+| `-pe` | `-an` | 238 words | pengrongrongan, penyatetan |
+| `-di` | `-n` | 182 words | disebabin, dianyarin |
+| `-ke` | `-n` | 180 words | kedoktoran, keaturan |
+| `-di` | `-in` | 172 words | disebabin, dianyarin |
+| `-ke` | `-an` | 167 words | kedoktoran, keaturan |
+| `-ng` | `-n` | 145 words | ngirimin, ngatasin |
+| `-ng` | `-in` | 140 words | ngirimin, ngatasin |
+| `-se` | `-a` | 50 words | serba, seninya |
+| `-pe` | `-a` | 47 words | pegihnja, perdananya |
+
+### 6.5 Recursive Morpheme Segmentation
+
+Using **Recursive Hierarchical Substitutability**, we decompose complex words into their constituent morphemes. This approach handles nested affixes (e.g., `prefix-prefix-root-suffix`).
+
+| Word | Suggested Split | Confidence | Stem |
+|------|-----------------|------------|------|
+| pengucapannya | **`pe-ng-ucap-an-nya`** | 9.0 | `ucap` |
+| kesaktiannya | **`ke-sakti-an-nya`** | 7.5 | `sakti` |
+| pengujungan | **`pe-ng-ujung-an`** | 7.5 | `ujung` |
+| dibilangin | **`di-bila-ng-in`** | 7.5 | `bila` |
+| pengrobahan | **`pe-ng-robah-an`** | 7.5 | `robah` |
+| kedaulatannya | **`ke-daulat-an-nya`** | 7.5 | `daulat` |
+| penggapaan | **`pe-ng-gapa-an`** | 7.5 | `gapa` |
+| diterjemahinnya | **`di-terjemah-in-nya`** | 7.5 | `terjemah` |
+| penggawéan | **`pe-ng-gawé-an`** | 7.5 | `gawé` |
+| sampingannya | **`sampi-ng-an-nya`** | 7.5 | `sampi` |
+| kebanyakannya | **`ke-banyak-an-nya`** | 7.5 | `banyak` |
+| dilindungin | **`di-lindu-ng-in`** | 7.5 | `lindu` |
+| dikeringin | **`di-ke-ring-in`** | 7.5 | `ring` |
+| kebalikannya | **`ke-balik-an-nya`** | 7.5 | `balik` |
+| dimaèninnya | **`di-maèn-in-nya`** | 7.5 | `maèn` |
+
+### 6.6 Linguistic Interpretation
+
+> **Automated Insight:**
+The language Betawi shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
+
+> **Note on Idiomaticity:** The high Idiomaticity Gap suggests a large number of frequent multi-word expressions or formulaic sequences that are statistically distinct from their component parts.
+
+---
+## 7. Summary & Recommendations
 
 ![Performance Dashboard](visualizations/performance_dashboard.png)
 
@@ -338,11 +552,12 @@ Below are text samples generated from each Markov chain model:
 
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
-| Tokenizer | **32k BPE** | Best compression (4.49x) with low UNK rate |
-| N-gram | **5-gram** | Lowest perplexity (288) |
-| Markov | **Context-4** | Highest predictability (96.5%) |
+| Tokenizer | **64k BPE** | Best compression (4.63x) |
+| N-gram | **2-gram** | Lowest perplexity (256) |
+| Markov | **Context-4** | Highest predictability (97.9%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
+
 ---
 ## Appendix: Metrics Glossary & Interpretation Guide
 
@@ -532,7 +747,8 @@ If you use these models in your research, please cite:
   author = {Kamali, Omar},
   title = {Wikilangs: Open NLP Models for Wikipedia Languages},
   year = {2025},
-  publisher = {HuggingFace},
+  doi = {10.5281/zenodo.18073153},
+  publisher = {Zenodo},
   url = {https://huggingface.co/wikilangs}
   institution = {Omneity Labs}
 }
@@ -548,7 +764,8 @@ MIT License - Free for academic and commercial use.
 - 🤗 Models: [huggingface.co/wikilangs](https://huggingface.co/wikilangs)
 - 📊 Data: [wikipedia-monthly](https://huggingface.co/datasets/omarkamali/wikipedia-monthly)
 - 👤 Author: [Omar Kamali](https://huggingface.co/omarkamali)
+- 🤝 Sponsor: [Featherless AI](https://featherless.ai)
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2025-12-28 02:18:06*
+*Report Date: 2026-01-03 18:42:18*

models/embeddings/aligned/bew_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9881405c6bfd25fab285c52fa84cf61230a22dac3d029ea903cc4ee51de74cba
+size 1031749150

models/embeddings/aligned/bew_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "bew", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bew_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5519fed88c0737dd92d5bfc338bdbac65a2f0249a74858399da99b311ee0f2c7
+size 65664

models/embeddings/aligned/bew_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bew",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 1788,
+  "vocab_size": 7447
+}

models/embeddings/aligned/bew_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f67005fdd1a646c4cab4fa8d00ad418d61e5a5997909d2692a20c93d633cb841
+size 258029854

models/embeddings/aligned/bew_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "bew", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bew_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce5dbde517db0b8f575eab1a3274aa737c012b45efddf2d0cb442f0096ca53f9
+size 4224

models/embeddings/aligned/bew_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bew",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 1788,
+  "vocab_size": 7447
+}

models/embeddings/aligned/bew_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b670eafad8c7deffbf21fdecd6fbcfc67b702800b97e90a03d7daeed98bc4c86
+size 515936286

models/embeddings/aligned/bew_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "bew", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bew_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ead9f472665f5a35d4374f24109e31f8ae69eb5bc410acf7dd40645aa7d82058
+size 16512

models/embeddings/aligned/bew_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bew",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 1788,
+  "vocab_size": 7447
+}

models/embeddings/monolingual/bew_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4335b040a535b0bd154cb31255037f3ed12d90b8411695a2d0d4ad1728d4dc87
-size 1032183649
+oid sha256:9881405c6bfd25fab285c52fa84cf61230a22dac3d029ea903cc4ee51de74cba
+size 1031749150

models/embeddings/monolingual/bew_128d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 128,
   "version": "monolingual",
   "training_params": {
-    "dim": 128,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 128
   },
-  "vocab_size": 7863
+  "vocab_size": 7447
 }

models/embeddings/monolingual/bew_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b2244ac75f23113c3ac11f29736e5e29c03b5e1e667fd5fd810422aa4e47da53
-size 258144865
+oid sha256:f67005fdd1a646c4cab4fa8d00ad418d61e5a5997909d2692a20c93d633cb841
+size 258029854

models/embeddings/monolingual/bew_32d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 32,
   "version": "monolingual",
   "training_params": {
-    "dim": 32,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 32
   },
-  "vocab_size": 7863
+  "vocab_size": 7447
 }

models/embeddings/monolingual/bew_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f38e536291d4ea1e5f53da361e36edb36a40a0edd8714d2da36cec0726e5b622
-size 516157793
+oid sha256:b670eafad8c7deffbf21fdecd6fbcfc67b702800b97e90a03d7daeed98bc4c86
+size 515936286

models/embeddings/monolingual/bew_64d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 64,
   "version": "monolingual",
   "training_params": {
-    "dim": 64,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 64
   },
-  "vocab_size": 7863
+  "vocab_size": 7447
 }

models/subword_markov/bew_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4bf262c92b34ec5661d8db620bc74ff3485373c52fa29ff8c7d2422a8136da30
-size 73122
+oid sha256:98e08c1e594701e44db187b0fea8553c31c5a2650d4d0337143ba0582a13e1ca
+size 70953

models/subword_markov/bew_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "bew",
-  "unique_contexts": 1469,
-  "total_transitions": 2526460
+  "unique_contexts": 1639,
+  "total_transitions": 2342625
 }

models/subword_markov/bew_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c6554a7de4144be71a218ddddea00779aef61fc58f934c3f0191ca8c80355c2f
-size 335416
+oid sha256:4579f280f945c8d062a950e4547166bf9a03594c2a3e93d696e59797932a621c
+size 304212

models/subword_markov/bew_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "bew",
-  "unique_contexts": 8912,
-  "total_transitions": 2523179
+  "unique_contexts": 8105,
+  "total_transitions": 2339359
 }

models/subword_markov/bew_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:11737a41163ad2d3dc78a88b3374643f4ab61b54f3acdd192526ea1a5ee7772c
-size 1094363
+oid sha256:98d72ab60b36827a0a2b03c0c604a02995c15de295a4b1f22e82b397b00b5c98
+size 1011297

models/subword_markov/bew_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "bew",
-  "unique_contexts": 41145,
-  "total_transitions": 2519898
+  "unique_contexts": 35638,
+  "total_transitions": 2336093
 }

models/subword_markov/bew_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16cda91fbc05488cedf0da6299c8e562dc38d26e9a3e36cfcdc4dd6d72338234
-size 2664959
+oid sha256:95ce2e7159b07b510279d2f42cb887b6e954dbaa740353c41154c042acb3c581
+size 2374737

models/subword_markov/bew_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "bew",
-  "unique_contexts": 141337,
-  "total_transitions": 2516617
+  "unique_contexts": 122163,
+  "total_transitions": 2332827
 }

models/subword_ngram/bew_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d11f42c7b2a133d183fa06b571114f5938df285f4b5141918f3b5efff0030d2
-size 38137
+oid sha256:5ed7a952fda6cd541de7eb0fd299625ce2b2758acc46e83c0cf659623b3f8210
+size 34201

models/subword_ngram/bew_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "bew",
-  "unique_ngrams": 2851,
-  "total_ngrams": 2526460
+  "unique_ngrams": 2508,
+  "total_ngrams": 2342625
 }

models/subword_ngram/bew_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48279f1d7817a148fc1255e0da6f55d1433148f1da9d1236709123c496ee8de8
-size 234049
+oid sha256:32b6c8714320f69bc8c5628d36482d0fc33c07d286f7c781527b80ff7ad46427
+size 204005

models/subword_ngram/bew_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "bew",
-  "unique_ngrams": 19026,
-  "total_ngrams": 2523179
+  "unique_ngrams": 16523,
+  "total_ngrams": 2339359
 }

models/subword_ngram/bew_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:258a4beb45713e7bf934e170923447b31b46b1ce23f58937f21f56f082969f23
-size 897268
+oid sha256:7604977f0b6c74871c004a352f1b6a38b2b0c0dfe4669c2f04692fe90d47058e
+size 803376

models/subword_ngram/bew_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "bew",
-  "unique_ngrams": 75515,
-  "total_ngrams": 2519898
+  "unique_ngrams": 66990,
+  "total_ngrams": 2336093
 }

models/subword_ngram/bew_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:642ccf861ef9688b285f4e80a179ee2d4d8dcfc320fb75b9c40c8360290d656f
+size 1537086

models/subword_ngram/bew_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "bew",
+  "unique_ngrams": 131412,
+  "total_ngrams": 2332827
+}

models/tokenizer/bew_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f3a1b400084e8023377501b802845cb1cf711d79ac5febd723de2b21cb129d8e
-size 503204
+oid sha256:242306074b42b2731865a15cce9984d5736ffc4699031020f06258b35683a505
+size 503923

models/tokenizer/bew_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e6232b0f1128785af7cc902e6ccd07e21b7b7b98c247d8d99404f7ad262866c1
-size 778728
+oid sha256:f9d4671fd36fd9b2ae145e0c81261494d435e4e3cacaa1d36f58cd1339019da7
+size 790627

models/tokenizer/bew_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a43c4038dd4030c982e33b34e05748d9a148f335d8f2efba64b7c0538588aed
-size 1366171
+oid sha256:99529365e2fc0dea414cae1cb7b20c351b8b62bccfe760eb5cda6c45d59e7f9b
+size 1375287

models/tokenizer/bew_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ef7e45246fddc1dfe48d9fe97af9c3c7abf4c75cfebb0b0038e553937f8383e4
-size 369839
+oid sha256:b81e52a146ace8aa44a481f72860065a471fff3ef540cf10543aa2199e9b3adc
+size 370756

models/vocabulary/bew_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ee57d5cd1890a06d6b99c4a39ea33a34f0319ef69950667ae5e96b854e97e04
-size 319583
+oid sha256:5fd039d507b57eb1e51b460f430b589898d9ee658bab288f4ad42ec5ce251291
+size 302381

models/vocabulary/bew_vocabulary_metadata.json

@@ -1,16 +1,17 @@
 {
   "language": "bew",
-  "vocabulary_size": 18978,
+  "vocabulary_size": 18200,
+  "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.11140219657569815,
+    "type_token_ratio": 0.11326404742870173,
     "coverage": {
-      "top_100": 0.38824255858956175,
-      "top_1000": 0.6505553445616538,
-      "top_5000": 0.8214500706614272,
-      "top_10000": 0.884600976083626
+      "top_100": 0.39197635158338345,
+      "top_1000": 0.6524853090255742,
+      "top_5000": 0.8229033596246144,
+      "top_10000": 0.8861755114711853
     },
-    "hapax_count": 24141,
-    "hapax_ratio": 0.5598691991929312,
-    "total_documents": 3281
+    "hapax_count": 23028,
+    "hapax_ratio": 0.5585524400892597,
+    "total_documents": 3266
   }
 }

models/word_markov/bew_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3cdf9da068448c4bfbde696835827ac01e4a769c1ec6964ab6bd9945242f7276
-size 1573718
+oid sha256:28749251ec198ef4ee8281befd0e291dcce482af6e56b0be6a53fc1d3f27fc49
+size 1479085

models/word_markov/bew_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "bew",
-  "unique_contexts": 43321,
-  "total_transitions": 502833
+  "unique_contexts": 41205,
+  "total_transitions": 360733
 }

models/word_markov/bew_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8057ab45a27f71e7f400d4c915c286a5afb7564f853cbd23531ee51ffb80882
-size 3691116
+oid sha256:a3505b2a73a013428ef6bceab4903f229062658d5b028c5aa86346e9466b035c
+size 3470528

models/word_markov/bew_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "bew",
-  "unique_contexts": 209140,
-  "total_transitions": 499553
+  "unique_contexts": 200219,
+  "total_transitions": 357467
 }