Upload all models and assets for dtp (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: dtp
-language_name: DTP
+language_name: Central Dusun
 language_family: austronesian_other
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-austronesian_other
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.589
+    value: 4.962
   - name: best_isotropy
     type: isotropy
-    value: 0.8746
+    value: 0.8679
   - name: vocabulary_size
     type: vocab
-    value: 31869
-generated: 2025-12-30
+    value: 0
+generated: 2026-01-04
 ---
 
-# DTP - Wikilangs Models
+# Central Dusun - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **DTP** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Central Dusun** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -44,12 +54,13 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 ### Models & Assets
 
 - Tokenizers (8k, 16k, 32k, 64k)
-- N-gram models (2, 3, 4-gram)
-- Markov chains (context of 1, 2, 3 and 4)
+- N-gram models (2, 3, 4, 5-gram)
+- Markov chains (context of 1, 2, 3, 4 and 5)
 - Subword N-gram and Markov chains
-- Embeddings in various sizes and dimensions
+- Embeddings in various sizes and dimensions (aligned and unaligned)
 - Language Vocabulary
 - Language Statistics
+
 ![Performance Dashboard](visualizations/performance_dashboard.png)
 
 ### Analysis and Evaluation
@@ -59,7 +70,8 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Summary & Recommendations](#6-summary--recommendations)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
+- [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
 
@@ -68,51 +80,57 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 ![Tokenizer Compression](visualizations/tokenizer_compression.png)
 
+![Tokenizer Fertility](visualizations/tokenizer_fertility.png)
+
+![Tokenizer OOV](visualizations/tokenizer_oov.png)
+
+![Total Tokens](visualizations/tokenizer_total_tokens.png)
+
 ### Results
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.730x | 3.70 | 0.0838% | 680,403 |
-| **16k** | 4.087x | 4.05 | 0.0918% | 620,992 |
-| **32k** | 4.371x | 4.33 | 0.0982% | 580,608 |
-| **64k** | 4.589x 🏆 | 4.55 | 0.1031% | 553,068 |
+| **8k** | 4.024x | 4.03 | 0.1643% | 595,784 |
+| **16k** | 4.420x | 4.42 | 0.1805% | 542,287 |
+| **32k** | 4.736x | 4.74 | 0.1934% | 506,176 |
+| **64k** | 4.962x 🏆 | 4.96 | 0.2026% | 483,109 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Kakamot (Inggilis: tools) nopo nga gunoon do papagaan karaja tokou monikid tadau...`
+**Sample 1:** `Boros Murut Timugon nopo nga boros di gunoon do Tulun Murut id Borneo. Sukuon`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁kakamot ▁( inggi lis : ▁to ol s ) ▁nopo ... (+10 more)` | 20 |
-| 16k | `▁kakamot ▁( inggilis : ▁to ols ) ▁nopo ▁nga ▁gunoon ... (+8 more)` | 18 |
-| 32k | `▁kakamot ▁( inggilis : ▁to ols ) ▁nopo ▁nga ▁gunoon ... (+8 more)` | 18 |
-| 64k | `▁kakamot ▁( inggilis : ▁tools ) ▁nopo ▁nga ▁gunoon ▁do ... (+6 more)` | 16 |
+| 8k | `▁boros ▁murut ▁tim ug on ▁nopo ▁nga ▁boros ▁di ▁gunoon ... (+7 more)` | 17 |
+| 16k | `▁boros ▁murut ▁tim ug on ▁nopo ▁nga ▁boros ▁di ▁gunoon ... (+7 more)` | 17 |
+| 32k | `▁boros ▁murut ▁tim ugon ▁nopo ▁nga ▁boros ▁di ▁gunoon ▁do ... (+6 more)` | 16 |
+| 64k | `▁boros ▁murut ▁timugon ▁nopo ▁nga ▁boros ▁di ▁gunoon ▁do ▁tulun ... (+5 more)` | 15 |
 
-**Sample 2:** `Boros Bukitan nopo nga iso kawo boros Punan id Kalimantan Barat, Indonesia om no...`
+**Sample 2:** `Suminundu nopo nga sinawaan di Kinoingan.Kitanak yolo do songulun tondu tolumis ...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁boros ▁buk itan ▁nopo ▁nga ▁iso ▁kawo ▁boros ▁p unan ... (+12 more)` | 22 |
-| 16k | `▁boros ▁buk itan ▁nopo ▁nga ▁iso ▁kawo ▁boros ▁p unan ... (+12 more)` | 22 |
-| 32k | `▁boros ▁buk itan ▁nopo ▁nga ▁iso ▁kawo ▁boros ▁p unan ... (+12 more)` | 22 |
-| 64k | `▁boros ▁bukitan ▁nopo ▁nga ▁iso ▁kawo ▁boros ▁punan ▁id ▁kalimantan ... (+10 more)` | 20 |
+| 8k | `▁sumin undu ▁nopo ▁nga ▁sin awaan ▁di ▁kino ingan . ... (+14 more)` | 24 |
+| 16k | `▁sumin undu ▁nopo ▁nga ▁sinawaan ▁di ▁kinoingan . k itanak ... (+11 more)` | 21 |
+| 32k | `▁sumin undu ▁nopo ▁nga ▁sinawaan ▁di ▁kinoingan . k itanak ... (+10 more)` | 20 |
+| 64k | `▁suminundu ▁nopo ▁nga ▁sinawaan ▁di ▁kinoingan . kitanak ▁yolo ▁do ... (+8 more)` | 18 |
 
-**Sample 3:** `Suminundu nopo nga sinawaan di Kinoingan.Kitanak yolo do songulun tondu tolumis ...`
+**Sample 3:** `Mongintob nopo nga nunu nopo iri kokomoi do ginumu, ginayo, sinodu toi winagat.`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁sumin undu ▁nopo ▁nga ▁sin awaan ▁di ▁kino ingan . ... (+16 more)` | 26 |
-| 16k | `▁sumin undu ▁nopo ▁nga ▁sin awaan ▁di ▁kino ingan . ... (+12 more)` | 22 |
-| 32k | `▁sumin undu ▁nopo ▁nga ▁sinawaan ▁di ▁kinoingan . kit anak ... (+10 more)` | 20 |
-| 64k | `▁suminundu ▁nopo ▁nga ▁sinawaan ▁di ▁kinoingan . kit anak ▁yolo ... (+9 more)` | 19 |
+| 8k | `▁mongin tob ▁nopo ▁nga ▁nunu ▁nopo ▁iri ▁kokomoi ▁do ▁ginumu ... (+7 more)` | 17 |
+| 16k | `▁mongintob ▁nopo ▁nga ▁nunu ▁nopo ▁iri ▁kokomoi ▁do ▁ginumu , ... (+6 more)` | 16 |
+| 32k | `▁mongintob ▁nopo ▁nga ▁nunu ▁nopo ▁iri ▁kokomoi ▁do ▁ginumu , ... (+6 more)` | 16 |
+| 64k | `▁mongintob ▁nopo ▁nga ▁nunu ▁nopo ▁iri ▁kokomoi ▁do ▁ginumu , ... (+6 more)` | 16 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.589x compression
-- **Lowest UNK Rate:** 8k with 0.0838% unknown tokens
+- **Best Compression:** 64k achieves 4.962x compression
+- **Lowest UNK Rate:** 8k with 0.1643% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -121,57 +139,111 @@ Below are sample sentences tokenized with each vocabulary size:
 
 ![N-gram Perplexity](visualizations/ngram_perplexity.png)
 
+![N-gram Unique](visualizations/ngram_unique.png)
+
 ![N-gram Coverage](visualizations/ngram_coverage.png)
 
 ### Results
 
-| N-gram | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
-|--------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | 9,628 🏆 | 13.23 | 26,102 | 16.3% | 38.1% |
-| **2-gram** | 268 🏆 | 8.07 | 3,536 | 68.7% | 99.0% |
-| **3-gram** | 16,586 | 14.02 | 30,760 | 10.5% | 27.5% |
-| **3-gram** | 2,315 | 11.18 | 24,789 | 26.8% | 71.8% |
-| **4-gram** | 28,313 | 14.79 | 40,735 | 6.7% | 18.5% |
-| **4-gram** | 12,602 | 13.62 | 114,796 | 13.5% | 40.4% |
+| N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
+|--------|---------|------------|---------|----------------|------------------|-------------------|
+| **2-gram** | Word | 7,224 | 12.82 | 18,432 | 17.6% | 40.2% |
+| **2-gram** | Subword | 227 🏆 | 7.82 | 2,665 | 72.6% | 99.5% |
+| **3-gram** | Word | 10,598 | 13.37 | 17,860 | 12.0% | 30.6% |
+| **3-gram** | Subword | 1,902 | 10.89 | 18,913 | 28.7% | 75.5% |
+| **4-gram** | Word | 17,687 | 14.11 | 21,653 | 5.2% | 18.7% |
+| **4-gram** | Subword | 10,332 | 13.33 | 90,801 | 14.5% | 42.9% |
+| **5-gram** | Word | 9,233 | 13.17 | 10,312 | 5.0% | 23.1% |
+| **5-gram** | Subword | 32,680 | 15.00 | 217,159 | 9.9% | 28.5% |
 
 ### Top 5 N-grams by Size
 
-**2-grams:**
+**2-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `nopo nga` | 11,657 |
+| 2 | `id suang` | 2,821 |
+| 3 | `toi ko` | 1,861 |
+| 4 | `ontok toun` | 1,828 |
+| 5 | `nga iso` | 1,049 |
+
+**3-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `nopo nga iso` | 951 |
+| 2 | `diti nopo nga` | 935 |
+| 3 | `id suang do` | 660 |
+| 4 | `nopo nga songulun` | 600 |
+| 5 | `nopo diti nga` | 439 |
+
+**4-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `nopo nga iso mantad` | 117 |
+| 2 | `nopo nga iso kawo` | 79 |
+| 3 | `nopo nga songulun mimingkono` | 75 |
+| 4 | `nopo nga kohompit no` | 71 |
+| 5 | `nopo nga iso pogun` | 70 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `archived from the original on` | 42 |
+| 2 | `toi ko lobi ointutunan sabaagi` | 34 |
+| 3 | `koposion pogulu om pondidikan nosusu` | 25 |
+| 4 | `toun uhu kono saluran tv` | 24 |
+| 5 | `mw parser output reflist lower` | 24 |
+
+**2-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `a n` | 132,420 |
+| 2 | `n _` | 100,917 |
+| 3 | `o _` | 92,031 |
+| 4 | `i _` | 88,621 |
+| 5 | `o n` | 79,747 |
+
+**3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `nopo nga` | 11,660 |
-| 2 | `, om` | 3,127 |
-| 3 | `id suang` | 2,820 |
-| 4 | `) ,` | 2,282 |
-| 5 | `ko '` | 2,260 |
+| 1 | `a n _` | 56,169 |
+| 2 | `d o _` | 34,236 |
+| 3 | `_ n o` | 33,345 |
+| 4 | `_ d o` | 32,858 |
+| 5 | `_ k o` | 28,766 |
 
-**3-grams:**
+**4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `toi ko '` | 1,571 |
-| 2 | `) nopo nga` | 946 |
-| 3 | `diti nopo nga` | 933 |
-| 4 | `nopo nga iso` | 879 |
-| 5 | `nopo nga '` | 863 |
+| 1 | `_ d o _` | 30,800 |
+| 2 | `_ i d _` | 22,452 |
+| 3 | `_ o m _` | 19,951 |
+| 4 | `_ n g a` | 17,310 |
+| 5 | `n o p o` | 15,354 |
 
-**4-grams:**
+**5-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `) nopo nga songulun` | 273 |
-| 2 | `https : / /` | 247 |
-| 3 | `) nopo nga iso` | 192 |
-| 4 | `. isido nopo nga` | 171 |
-| 5 | `sundung do ingkaa ,` | 163 |
+| 1 | `_ n g a _` | 14,567 |
+| 2 | `_ n o p o` | 14,303 |
+| 3 | `n o p o _` | 14,096 |
+| 4 | `o n t o k` | 12,540 |
+| 5 | `n t o k _` | 12,488 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram with 268
+- **Best Perplexity:** 2-gram (subword) with 227
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~40% of corpus
+- **Coverage:** Top-1000 patterns cover ~29% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -179,55 +251,86 @@ Below are sample sentences tokenized with each vocabulary size:
 
 ![Markov Entropy](visualizations/markov_entropy.png)
 
+![Markov Contexts](visualizations/markov_contexts.png)
+
 ![Markov Branching](visualizations/markov_branching.png)
 
 ### Results
 
-| Context | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
-|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | 0.7421 | 1.673 | 5.24 | 79,291 | 25.8% |
-| **1** | 1.0541 | 2.076 | 6.17 | 1,907 | 0.0% |
-| **2** | 0.3235 | 1.251 | 1.79 | 414,760 | 67.7% |
-| **2** | 0.8143 | 1.758 | 4.77 | 11,770 | 18.6% |
-| **3** | 0.1058 | 1.076 | 1.18 | 740,308 | 89.4% |
-| **3** | 0.8194 | 1.765 | 3.86 | 56,142 | 18.1% |
-| **4** | 0.0353 🏆 | 1.025 | 1.05 | 874,067 | 96.5% |
-| **4** | 0.6196 🏆 | 1.536 | 2.60 | 216,827 | 38.0% |
+| Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
+|---------|---------|-------------|------------|------------------|-----------------|----------------|
+| **1** | Word | 0.8540 | 1.808 | 5.51 | 70,711 | 14.6% |
+| **1** | Subword | 0.8991 | 1.865 | 5.16 | 1,986 | 10.1% |
+| **2** | Word | 0.2712 | 1.207 | 1.62 | 388,589 | 72.9% |
+| **2** | Subword | 0.6820 | 1.604 | 4.13 | 10,241 | 31.8% |
+| **3** | Word | 0.0811 | 1.058 | 1.13 | 628,158 | 91.9% |
+| **3** | Subword | 0.7746 | 1.711 | 3.85 | 42,293 | 22.5% |
+| **4** | Word | 0.0237 🏆 | 1.017 | 1.03 | 709,279 | 97.6% |
+| **4** | Subword | 0.6516 | 1.571 | 2.76 | 162,763 | 34.8% |
+
+### Generated Text Samples (Word-based)
+
+Below are text samples generated from each word-based Markov chain model:
+
+**Context Size 1:**
+
+1. `do tasu piipiro posis nopo nga bagas menteri malaysia toi ko 7 3w 7 808 güzelbahçe`
+2. `id boros sweden maamaso timpu pogulu nosusu i nopo nga okito nogi i rajaa do amu`
+3. `om papaharo sikul takawas id keningau diti nga kohompit om gisom pinoposiliu do dudumagang maritim m...`
+
+**Context Size 2:**
+
+1. `nopo nga okito id posorili do kuil kuil bongunan bongunan winonsoi o kinoyonon diti galeri sukuon pa...`
+2. `id suang pambalajalan loolobi id gana do sains sosial om ekonomi mogigion do pulau bali winonsoi o`
+3. `toi ko bandar raya santiago gurun atacama ii gersang id utara chile nopo nga kosoruan ointutunan sab...`
+
+**Context Size 3:**
+
+1. `nopo nga iso kakadayan komponen kalas ko 5 id kointayadan do 50 tondu yahudi di bobos boroson id`
+2. `diti nopo nga kiwaa totos okuri nopo nga kirati do tudan udan talasu om i bobos poinwagu nopo`
+3. `id suang do watas tenom om id siriba kotoinaan do upis watas keningau di laid abaabayan dii nopo`
+
+**Context Size 4:**
+
+1. `nopo nga iso mantad tolu puruan tinimungan slav kosilahon ii kakal po do pharo ii suai nopo nga monu...`
+2. `nopo nga iso kawo boros dayak i popohompit do duo dialek daro om matu dialek mantad boros austronesi...`
+3. `nopo nga songulun mimingkono di abantung kopio maya piipiro film miagal ko x men apocalypse om nogi ...`
 
-### Generated Text Samples
 
-Below are text samples generated from each Markov chain model:
+### Generated Text Samples (Subword-based)
+
+Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `, but beethoven . delear , hello , 13 , popolombus hiis . … 배드민턴 안세영`
-2. `. biografi kopointutunan maamaso momongo kalaja inus kapamansayan ontok momuhau 24 : two , 30 koumat...`
-3. `do urusan om gana ' s love you best contemporary dangdut pesisir barat om koubasanan id`
+1. `_2_,_suhyl_palal`
+2. `aheacasomomoid_p`
+3. `ombaaiayosiesili`
 
 **Context Size 2:**
 
-1. `nopo nga iso kakadayan topuhawang om totos totuo ' id korea selatan id poimpuli a . s`
-2. `, om nogi minog mamagayat rahi songulun mokiikinobos doktor falsafah ( jawi ) do fosforus , kalsium`
-3. `id suang gana noos dau ponokoguas do cavell . [ meragukan ] soginumuan soriuk do ponoriukan fenomena`
+1. `an_gan_ka_kopoko_`
+2. `n_mek_koudions_gr`
+3. `o_dukul_bihaguluh`
 
 **Context Size 3:**
 
-1. `toi ko ' ointutunan sabaagi jaclyn victor ( nosusu ontok 23 mahas , 2000 ) . the consequences`
-2. `) nopo nga songulun do kiraha mirolot kadazan - polish ii nosusu ontok kurun ko - 13 id`
-3. `diti nopo nga i che zan ii kiraha do naralatan sina om india muslim mantad do pulau pinang`
+1. `an_abaagu_di_aut"_`
+2. `do_sukuon_debutang`
+3. `_nokobol_kopo_ngam`
 
 **Context Size 4:**
 
-1. `) nopo nga songulun monondig om mimingkono tinaru turki ii mobi mantad awakening records . kopointut...`
-2. `https : / / www . jagranjosh . com / general - knowledge / national - sports - day`
-3. `) nopo nga iso kalab buul sipak arab saudi i poingimpou id riyadh . noturidong o kalab diti ontok`
+1. `_do_ponuan_chillage`
+2. `_id_sabaagi_gisom_n`
+3. `_om_institud_5.11-3`
 
 
 ### Key Findings
 
-- **Best Predictability:** Context-4 with 96.5% predictability
+- **Best Predictability:** Context-4 (word) with 97.6% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (216,827 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (162,763 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -243,64 +346,64 @@ Below are text samples generated from each Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 31,869 |
-| Total Tokens | 750,543 |
-| Mean Frequency | 23.55 |
+| Vocabulary Size | 30,571 |
+| Total Tokens | 714,971 |
+| Mean Frequency | 23.39 |
 | Median Frequency | 4 |
-| Frequency Std Dev | 317.13 |
+| Frequency Std Dev | 322.81 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | do | 30,943 |
-| 2 | id | 22,628 |
-| 3 | om | 20,012 |
-| 4 | nga | 15,887 |
-| 5 | nopo | 14,215 |
-| 6 | di | 13,706 |
-| 7 | i | 9,800 |
-| 8 | mantad | 7,464 |
-| 9 | ontok | 6,786 |
-| 10 | sabaagi | 5,795 |
+| 1 | do | 30,939 |
+| 2 | id | 22,604 |
+| 3 | om | 20,001 |
+| 4 | nga | 15,882 |
+| 5 | nopo | 14,210 |
+| 6 | di | 13,677 |
+| 7 | i | 9,637 |
+| 8 | mantad | 7,460 |
+| 9 | ontok | 6,784 |
+| 10 | sabaagi | 5,793 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | 43500 | 2 |
-| 2 | 95250 | 2 |
-| 3 | 31389 | 2 |
-| 4 | 87167 | 2 |
-| 5 | sisimbarpulou | 2 |
-| 6 | 626417 | 2 |
-| 7 | 757833 | 2 |
-| 8 | 72194 | 2 |
-| 9 | 64972 | 2 |
+| 1 | nın | 2 |
+| 2 | tarihçesi | 2 |
+| 3 | paü | 2 |
+| 4 | eğitim | 2 |
+| 5 | dergisi | 2 |
+| 6 | sayı | 2 |
+| 7 | mongumang | 2 |
+| 8 | mikattiwang | 2 |
+| 9 | sisimbarpulou | 2 |
 | 10 | koz | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 1.0562 |
-| R² (Goodness of Fit) | 0.993179 |
+| Zipf Coefficient | 1.0496 |
+| R² (Goodness of Fit) | 0.994075 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 39.9% |
-| Top 1,000 | 65.4% |
-| Top 5,000 | 84.2% |
-| Top 10,000 | 90.9% |
+| Top 100 | 41.6% |
+| Top 1,000 | 66.1% |
+| Top 5,000 | 84.5% |
+| Top 10,000 | 91.2% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9932 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 39.9% of corpus
-- **Long Tail:** 21,869 words needed for remaining 9.1% coverage
+- **Zipf Compliance:** R²=0.9941 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 41.6% of corpus
+- **Long Tail:** 20,571 words needed for remaining 8.8% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -313,24 +416,130 @@ Below are text samples generated from each Markov chain model:
 
 ![t-SNE Sentences](visualizations/tsne_sentences.png)
 
-### Model Comparison
 
-| Model | Vocab Size | Dimension | Avg Norm | Std Norm | Isotropy |
-|-------|------------|-----------|----------|----------|----------|
-| **mono_32d** | 15,036 | 32 | 3.350 | 0.651 | 0.8746 🏆 |
-| **mono_64d** | 15,036 | 64 | 3.723 | 0.623 | 0.7529 |
-| **mono_128d** | 15,036 | 128 | 3.904 | 0.607 | 0.3563 |
-| **embeddings_enhanced** | 0 | 0 | 0.000 | 0.000 | 0.0000 |
+### 5.1 Cross-Lingual Alignment
+
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
+
+
+### 5.2 Model Comparison
+
+| Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
+|-------|-----------|----------|------------------|---------------|----------------|
+| **mono_32d** | 32 | 0.8679 🏆 | 0.3272 | N/A | N/A |
+| **mono_64d** | 64 | 0.7620 | 0.2632 | N/A | N/A |
+| **mono_128d** | 128 | 0.3462 | 0.2417 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8679 | 0.3226 | 0.0560 | 0.2820 |
+| **aligned_64d** | 64 | 0.7620 | 0.2720 | 0.1060 | 0.3860 |
+| **aligned_128d** | 128 | 0.3462 | 0.2427 | 0.2020 | 0.5260 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.8746 (more uniform distribution)
-- **Dimension Trade-off:** Higher dimensions capture more semantics but reduce isotropy
-- **Vocabulary Coverage:** All models cover 15,036 words
-- **Recommendation:** 100d for balanced semantic capture and efficiency
+- **Best Isotropy:** mono_32d with 0.8679 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.2782. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 20.2% R@1 in cross-lingual retrieval.
+- **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
-## 6. Summary & Recommendations
+## 6.  Morphological Analysis (Experimental)
+
+This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
+
+### 6.1 Productivity & Complexity
+
+| Metric | Value | Interpretation | Recommendation |
+|--------|-------|----------------|----------------|
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **-0.189** | Low formulaic content | - |
+
+### 6.2 Affix Inventory (Productive Units)
+
+These are the most productive prefixes and suffixes identified by sampling the vocabulary for global substitutability patterns. A unit is considered an affix if stripping it leaves a valid stem that appears in other contexts.
+
+#### Productive Prefixes
+| Prefix | Examples |
+|--------|----------|
+| `-po` | poinkilong, pointounda, poninong |
+| `-ko` | kopogonuan, kontinjen, kokomoi |
+| `-mo` | monongkuyaan, mongingit, mohd |
+| `-mi` | mind, millennium, minsingumbal |
+| `-ma` | maru, many, matter |
+
+#### Productive Suffixes
+| Suffix | Examples |
+|--------|----------|
+| `-n` | louson, sukun, monongkuyaan |
+| `-an` | monongkuyaan, kopogonuan, keahlian |
+| `-on` | louson, southampton, unsubon |
+| `-ng` | poinkilong, skateboarding, dropping |
+
+### 6.3 Bound Stems (Lexical Roots)
+
+Bound stems are high-frequency subword units that are semantically cohesive but rarely appear as standalone words. These often correspond to the 'core' of a word that requires inflection or derivation to be valid.
+
+| Stem | Cohesion | Substitutability | Examples |
+|------|----------|------------------|----------|
+| `anga` | 1.64x | 146 contexts | ganga, tanga, manga |
+| `ngan` | 1.88x | 34 contexts | songan, jangan, dengan |
+| `oros` | 2.02x | 26 contexts | boros, oroso, doros |
+| `anta` | 1.48x | 88 contexts | banta, manta, antad |
+| `boro` | 2.19x | 19 contexts | boros, oboros, borough |
+| `ongu` | 1.63x | 50 contexts | tongue, tongus, mongua |
+| `impu` | 1.96x | 24 contexts | limpu, timpu, limput |
+| `mont` | 1.81x | 26 contexts | monto, montk, monte |
+| `ampa` | 1.48x | 47 contexts | campa, gampa, rampa |
+| `uang` | 1.59x | 33 contexts | huang, duang, ruang |
+| `ogun` | 1.79x | 21 contexts | oguno, pogun, koguno |
+| `mpai` | 1.95x | 13 contexts | ampai, rumpai, mimpai |
+
+### 6.4 Affix Compatibility (Co-occurrence)
+
+This table shows which prefixes and suffixes most frequently co-occur on the same stems, revealing the 'stacking' rules of the language's morphology.
+
+| Prefix | Suffix | Frequency | Examples |
+|--------|--------|-----------|----------|
+| `-ko` | `-n` | 164 words | kolintuhunan, koyomutan |
+| `-po` | `-n` | 148 words | poimpohon, porundangan |
+| `-ko` | `-an` | 121 words | kolintuhunan, koyomutan |
+| `-po` | `-an` | 109 words | porundangan, pomutulan |
+| `-po` | `-on` | 39 words | poimpohon, potingkodon |
+| `-ko` | `-on` | 37 words | kohinoon, kosogubon |
+| `-mi` | `-ng` | 29 words | minanamong, minongisonong |
+| `-mi` | `-n` | 23 words | million, miimpohon |
+| `-mo` | `-ng` | 22 words | momoguring, moyang |
+| `-po` | `-ng` | 16 words | poring, poning |
+
+### 6.5 Recursive Morpheme Segmentation
+
+Using **Recursive Hierarchical Substitutability**, we decompose complex words into their constituent morphemes. This approach handles nested affixes (e.g., `prefix-prefix-root-suffix`).
+
+| Word | Suggested Split | Confidence | Stem |
+|------|-----------------|------------|------|
+| kopomolobusan | **`ko-po-mo-lobus-an`** | 9.0 | `lobus` |
+| popokobong | **`po-po-ko-bong`** | 7.5 | `bong` |
+| pomokritik | **`po-mo-kritik`** | 6.0 | `kritik` |
+| popobibas | **`po-po-bibas`** | 6.0 | `bibas` |
+| momooboros | **`mo-mo-oboros`** | 6.0 | `oboros` |
+| mamagakom | **`ma-ma-gakom`** | 6.0 | `gakom` |
+| pomodolinan | **`po-mo-dolin-an`** | 4.5 | `dolin` |
+| koingkuri | **`ko-ingkuri`** | 4.5 | `ingkuri` |
+| tungkusan | **`tungkus-an`** | 4.5 | `tungkus` |
+| pengurusan | **`pengurus-an`** | 4.5 | `pengurus` |
+| kopogisuusuayan | **`ko-po-gisuusuay-an`** | 4.5 | `gisuusuay` |
+| pesisiran | **`pesisir-an`** | 4.5 | `pesisir` |
+| kopomoogian | **`ko-po-mo-ogian`** | 4.5 | `ogian` |
+| pomudagangan | **`po-mudaga-ng-an`** | 4.5 | `mudaga` |
+| pomobodilan | **`po-mo-bodil-an`** | 4.5 | `bodil` |
+
+### 6.6 Linguistic Interpretation
+
+> **Automated Insight:**
+The language Central Dusun shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
+
+---
+## 7. Summary & Recommendations
 
 ![Performance Dashboard](visualizations/performance_dashboard.png)
 
@@ -338,11 +547,12 @@ Below are text samples generated from each Markov chain model:
 
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
-| Tokenizer | **32k BPE** | Best compression (4.59x) with low UNK rate |
-| N-gram | **5-gram** | Lowest perplexity (268) |
-| Markov | **Context-4** | Highest predictability (96.5%) |
+| Tokenizer | **64k BPE** | Best compression (4.96x) |
+| N-gram | **2-gram** | Lowest perplexity (227) |
+| Markov | **Context-4** | Highest predictability (97.6%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
+
 ---
 ## Appendix: Metrics Glossary & Interpretation Guide
 
@@ -532,7 +742,8 @@ If you use these models in your research, please cite:
   author = {Kamali, Omar},
   title = {Wikilangs: Open NLP Models for Wikipedia Languages},
   year = {2025},
-  publisher = {HuggingFace},
+  doi = {10.5281/zenodo.18073153},
+  publisher = {Zenodo},
   url = {https://huggingface.co/wikilangs}
   institution = {Omneity Labs}
 }
@@ -548,7 +759,8 @@ MIT License - Free for academic and commercial use.
 - 🤗 Models: [huggingface.co/wikilangs](https://huggingface.co/wikilangs)
 - 📊 Data: [wikipedia-monthly](https://huggingface.co/datasets/omarkamali/wikipedia-monthly)
 - 👤 Author: [Omar Kamali](https://huggingface.co/omarkamali)
+- 🤝 Sponsor: [Featherless AI](https://featherless.ai)
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2025-12-30 08:36:47*
+*Report Date: 2026-01-04 02:42:58*

models/embeddings/aligned/dtp_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef8c92c34c5389ef126c85a7dbfebc6df2ad2da9260a085f1dcfdff1699073a6
+size 1038885555

models/embeddings/aligned/dtp_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "dtp", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/dtp_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:261f45003a3ff551ce887a6bd36c3baa22a7188419587a6a39c1ccc6aa87d04f
+size 65664

models/embeddings/aligned/dtp_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "dtp",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 5929,
+  "vocab_size": 14302
+}

models/embeddings/aligned/dtp_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:484a1fe0155c7f9986f9ee1ae8e8b474143a65746ccf923c6d3db3855b08dd50
+size 259901619

models/embeddings/aligned/dtp_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "dtp", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/dtp_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2e64e5308e85adac861732e68ce3904f7af20969baa3c44b28062e763101fba
+size 4224

models/embeddings/aligned/dtp_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "dtp",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 5929,
+  "vocab_size": 14302
+}

models/embeddings/aligned/dtp_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:453663b596b40562830f834a81e45581e60a99d44b931538b689055f8f6b8dd1
+size 519562931

models/embeddings/aligned/dtp_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "dtp", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/dtp_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:819a56cc945dca6d803974102b732c51ca35de6eb1756123670c31cae5dedc51
+size 16512

models/embeddings/aligned/dtp_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "dtp",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 5929,
+  "vocab_size": 14302
+}

models/embeddings/monolingual/dtp_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:64368ca20c13e820bd6729b2433d8becff40fa91dfa8b2fe85413e8bfa4fc976
-size 1039649725
+oid sha256:ef8c92c34c5389ef126c85a7dbfebc6df2ad2da9260a085f1dcfdff1699073a6
+size 1038885555

models/embeddings/monolingual/dtp_128d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 128,
   "version": "monolingual",
   "training_params": {
-    "dim": 128,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 128
   },
-  "vocab_size": 15036
+  "vocab_size": 14302
 }

models/embeddings/monolingual/dtp_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8444cc0255158a6e9f9b000f28544827397224b956952639e668e15fa849737d
-size 260102077
+oid sha256:484a1fe0155c7f9986f9ee1ae8e8b474143a65746ccf923c6d3db3855b08dd50
+size 259901619

models/embeddings/monolingual/dtp_32d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 32,
   "version": "monolingual",
   "training_params": {
-    "dim": 32,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 32
   },
-  "vocab_size": 15036
+  "vocab_size": 14302
 }

models/embeddings/monolingual/dtp_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cbd05a85ec9ea45c19d599c07b616e901cb9e0c2a8c529d50eb66978765fccbf
-size 519951293
+oid sha256:453663b596b40562830f834a81e45581e60a99d44b931538b689055f8f6b8dd1
+size 519562931

models/embeddings/monolingual/dtp_64d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 64,
   "version": "monolingual",
   "training_params": {
-    "dim": 64,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 64
   },
-  "vocab_size": 15036
+  "vocab_size": 14302
 }

models/subword_markov/dtp_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:026dfb688d83cbd8de75ac6f7f2d984fdb2e4cf577fcbb30c1b82a5fda4387f6
-size 90282
+oid sha256:c689cc0e9e37a6641e97673744cf0aa8458bc5c144a694bb6f6126f2a1fd9ac3
+size 82716

models/subword_markov/dtp_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "dtp",
-  "unique_contexts": 1907,
-  "total_transitions": 5244532
+  "unique_contexts": 1986,
+  "total_transitions": 4891989
 }

models/subword_markov/dtp_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:10f57d9ab5ff100fa24fc205370ce9ef284a07c6cf89a4ed39379ca45b14cee8
-size 442873
+oid sha256:e11ab1f6270f3d83b9accbdb1caa49db8f09b49b4107cb2ecd426f4e07a67a87
+size 357574

models/subword_markov/dtp_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "dtp",
-  "unique_contexts": 11770,
-  "total_transitions": 5242606
+  "unique_contexts": 10241,
+  "total_transitions": 4890072
 }

models/subword_markov/dtp_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d3e6fc10106e41d05bdcdf2c8bfd2a4da0f1fb5798fcbfdcc25a4d320463f74
-size 1631340
+oid sha256:2cc1b59a67a565880863c396871ea1e552ba11d3a1ce3c24b30859de39a3f7a7
+size 1290349

models/subword_markov/dtp_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "dtp",
-  "unique_contexts": 56142,
-  "total_transitions": 5240680
+  "unique_contexts": 42293,
+  "total_transitions": 4888155
 }

models/subword_markov/dtp_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f641243fcd9d959160e9be0fc8dbf51c577f71bd06249a96158b942ca111017
-size 4387258
+oid sha256:98e90afa361bfa3d24fd0391063c2ecd178e827eaff193eb36e54565610a0c6b
+size 3460742

models/subword_markov/dtp_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "dtp",
-  "unique_contexts": 216827,
-  "total_transitions": 5238754
+  "unique_contexts": 162763,
+  "total_transitions": 4886238
 }

models/subword_ngram/dtp_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b1a6dd61fce8e08269ecd821a19e43e8d3eca2bd245d96fdba3240ec081b5a26
-size 44336
+oid sha256:d8386f25b551413e623afb1ee5f3baa26b0589ec02c8b3eb088530731d3f1640
+size 35393

models/subword_ngram/dtp_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "dtp",
-  "unique_ngrams": 3536,
-  "total_ngrams": 5244532
+  "unique_ngrams": 2665,
+  "total_ngrams": 4891989
 }

models/subword_ngram/dtp_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c3dccc5541e601e231271163bb00f5f420140413659293af0cec60bb86f1d18a
-size 293148
+oid sha256:5b698de07a91cd02f32dce2f0a55eca95a01cbac330ba41d817fcef280318fce
+size 227164

models/subword_ngram/dtp_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "dtp",
-  "unique_ngrams": 24789,
-  "total_ngrams": 5242606
+  "unique_ngrams": 18913,
+  "total_ngrams": 4890072
 }

models/subword_ngram/dtp_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a628aae7320eb63538107a02caad103d0f40b5059535b7c5cc4e886ac21b2944
-size 1264464
+oid sha256:8832be2c290f584206541b682bfad0ff537420bf107ee94faf3e93ec6f112875
+size 1016286

models/subword_ngram/dtp_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "dtp",
-  "unique_ngrams": 114796,
-  "total_ngrams": 5240680
+  "unique_ngrams": 90801,
+  "total_ngrams": 4888155
 }

models/subword_ngram/dtp_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4571ff87918d80ffa59d68274da2f88a69067c04b91c900db5348bbe73d16e6
+size 2445166

models/subword_ngram/dtp_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "dtp",
+  "unique_ngrams": 217159,
+  "total_ngrams": 4886238
+}

models/tokenizer/dtp_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2863a29b8952092740058fbb7898f210df9c20d796dc716afef3efcaccdf5a95
-size 504001
+oid sha256:4dceead38cfa328598cf6590d260b180e4c6e1336b1bac03196c98eef591678c
+size 505828

models/tokenizer/dtp_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:412a430f2c94cb19c91aa212b6084bca872656a70e5b58d960a0fe0d507630eb
-size 773984
+oid sha256:ce8e0988abdbca791e274ed96ac8142860d79460547dc20c42086a9f4bcad0c7
+size 778468

models/tokenizer/dtp_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a568ec26551d4463e96d413d136b262ce05e01fe03c7dbc4eea0abb1033d0b59
-size 1342459
+oid sha256:1577ca3b5bdae4aa585e5bfbde482a04393b42570719d727347b14d03585fcbb
+size 1355760

models/tokenizer/dtp_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fcd5170a46936eba2d4f97ca832c95ea3722e47552200560b4f8b1ed1c9c1cac
-size 370293
+oid sha256:36ea927a96c120c1ce16e6c009bfe36ff392efd27af725fc98db889699bee0cd
+size 371878

models/vocabulary/dtp_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:beef90971c97587844d6f5c135be62bbe41b5c2c7b983bc121f0a5462757881d
-size 540408
+oid sha256:2a4f8129665d48c97d3572e4cf1c1a8caae33fbc15fa51f7acd50817c717b467
+size 515055

models/vocabulary/dtp_vocabulary_metadata.json

@@ -1,16 +1,17 @@
 {
   "language": "dtp",
-  "vocabulary_size": 31869,
+  "vocabulary_size": 30571,
+  "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.09918250392954643,
+    "type_token_ratio": 0.09374159982944757,
     "coverage": {
-      "top_100": 0.3756633851507016,
-      "top_1000": 0.614794146918659,
-      "top_5000": 0.79167763430024,
-      "top_10000": 0.8552034214002974
+      "top_100": 0.3939244669905574,
+      "top_1000": 0.626013482646158,
+      "top_5000": 0.8002378200009799,
+      "top_10000": 0.8636600180351248
     },
-    "hapax_count": 47259,
-    "hapax_ratio": 0.5972474977252047,
-    "total_documents": 1926
+    "hapax_count": 40222,
+    "hapax_ratio": 0.5681635189919907,
+    "total_documents": 1917
   }
 }

models/word_markov/dtp_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ebe4ab1bbfc38f8ff2eb5c585129c66bc368d8f9a24070550e5ae8ddb87a2abd
-size 3083139
+oid sha256:6c316510488a25c381b09878d446683f932d124c72ec4c271448efba25a8baec
+size 2874301

models/word_markov/dtp_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "dtp",
-  "unique_contexts": 79291,
-  "total_transitions": 967120
+  "unique_contexts": 70711,
+  "total_transitions": 753276
 }

models/word_markov/dtp_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:114a5ca2c5b0c7fdfb0d1addcc8dc599458316d5b6aa10441a3521b7f6af14dd
-size 7641242
+oid sha256:ab2f25e629b0080d735455721dd4e9dbe6ac21a644af6d9d1ec1918143c90f13
+size 6945374

models/word_markov/dtp_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "dtp",
-  "unique_contexts": 414760,
-  "total_transitions": 965194
+  "unique_contexts": 388589,
+  "total_transitions": 751359
 }