Upload all models and assets for ab (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: ab
-language_name: AB
+language_name: Abkhazian
 language_family: caucasian_northwest
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-caucasian_northwest
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -26,17 +36,17 @@ metrics:
     value: 4.193
   - name: best_isotropy
     type: isotropy
-    value: 0.8185
+    value: 0.8394
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# AB - Wikilangs Models
+# Abkhazian - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **AB** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Abkhazian** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.304x | 3.31 | 0.1494% | 223,505 |
-| **16k** | 3.652x | 3.66 | 0.1652% | 202,175 |
-| **32k** | 3.908x | 3.91 | 0.1768% | 188,952 |
-| **64k** | 4.193x 🏆 | 4.20 | 0.1897% | 176,103 |
+| **8k** | 3.306x | 3.31 | 0.1493% | 223,032 |
+| **16k** | 3.654x | 3.66 | 0.1650% | 201,823 |
+| **32k** | 3.910x | 3.92 | 0.1766% | 188,563 |
+| **64k** | 4.193x 🏆 | 4.20 | 0.1893% | 175,871 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Ψ, ψ — бырзентәи аҩыратә нбан. Азхьарԥшқәа Graphemica (Ψ) Graphemica (ψ) аҩыратә...`
+**Sample 1:** `Ѳ, ѳ — кириллтәи аҩыратә архаикатә иажәхьоу нбан. Азхьарԥшқәа Graphemica (Ѳ) Gra...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁ ψ , ▁ ψ ▁— ▁бырзентәи ▁аҩыратә ▁нбан . ... (+11 more)` | 21 |
-| 16k | `▁ψ , ▁ψ ▁— ▁бырзентәи ▁аҩыратә ▁нбан . ▁азхьарԥшқәа ▁graphemica ... (+9 more)` | 19 |
-| 32k | `▁ψ , ▁ψ ▁— ▁бырзентәи ▁аҩыратә ▁нбан . ▁азхьарԥшқәа ▁graphemica ... (+9 more)` | 19 |
-| 64k | `▁ψ , ▁ψ ▁— ▁бырзентәи ▁аҩыратә ▁нбан . ▁азхьарԥшқәа ▁graphemica ... (+9 more)` | 19 |
+| 8k | `▁ ѳ , ▁ ѳ ▁— ▁кириллтәи ▁аҩыратә ▁архаикатә ▁иажәхьоу ... (+11 more)` | 21 |
+| 16k | `▁ѳ , ▁ѳ ▁— ▁кириллтәи ▁аҩыратә ▁архаикатә ▁иажәхьоу ▁нбан . ... (+9 more)` | 19 |
+| 32k | `▁ѳ , ▁ѳ ▁— ▁кириллтәи ▁аҩыратә ▁архаикатә ▁иажәхьоу ▁нбан . ... (+9 more)` | 19 |
+| 64k | `▁ѳ , ▁ѳ ▁— ▁кириллтәи ▁аҩыратә ▁архаикатә ▁иажәхьоу ▁нбан . ... (+9 more)` | 19 |
 
-**Sample 2:** `Амолдав бызшәа, амолдаван бызшәа (limba moldovenească, лимба молдовеняскэ) Азгәа...`
+**Sample 2:** `Скуо-Уелли Winter Olympics, Jeux olympiques d'hiver de - аӡынтәи Олимпиадатә хәм...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁ам ол да в ▁бызшәа , ▁ам ол да ван ... (+24 more)` | 34 |
-| 16k | `▁амол дав ▁бызшәа , ▁амол да ван ▁бызшәа ▁( l ... (+20 more)` | 30 |
-| 32k | `▁амолдав ▁бызшәа , ▁амол да ван ▁бызшәа ▁( l imba ... (+13 more)` | 23 |
-| 64k | `▁амолдав ▁бызшәа , ▁амолдаван ▁бызшәа ▁( limba ▁moldoveneasc ă , ... (+5 more)` | 15 |
+| 8k | `▁с ку о - у елли ▁winter ▁olympics , ▁jeux ... (+12 more)` | 22 |
+| 16k | `▁с ку о - у елли ▁winter ▁olympics , ▁jeux ... (+12 more)` | 22 |
+| 32k | `▁с ку о - уелли ▁winter ▁olympics , ▁jeux ▁olympiques ... (+11 more)` | 21 |
+| 64k | `▁скуо - уелли ▁winter ▁olympics , ▁jeux ▁olympiques ▁d ' ... (+9 more)` | 19 |
 
-**Sample 3:** `Ардешен () – Ҭырқтәыла ақалақь. Иаланхо . Ахьарԥшқәа ақалақьқәа`
+**Sample 3:** `Ж, ж — кириллтәи аҩыратә нбан. Азхьарԥшқәа Graphemica (Ж) Graphemica (ж)`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁ар деш ен ▁() ▁– ▁ҭырқтәыла ▁ақалақь . ▁иаланхо ▁. ... (+2 more)` | 12 |
-| 16k | `▁ар деш ен ▁() ▁– ▁ҭырқтәыла ▁ақалақь . ▁иаланхо ▁. ... (+2 more)` | 12 |
-| 32k | `▁ардешен ▁() ▁– ▁ҭырқтәыла ▁ақалақь . ▁иаланхо ▁. ▁ахьарԥшқәа ▁ақалақьқәа` | 10 |
-| 64k | `▁ардешен ▁() ▁– ▁ҭырқтәыла ▁ақалақь . ▁иаланхо ▁. ▁ахьарԥшқәа ▁ақалақьқәа` | 10 |
+| 8k | `▁ж , ▁ж ▁— ▁кириллтәи ▁аҩыратә ▁нбан . ▁азхьарԥшқәа ▁graphemica ... (+7 more)` | 17 |
+| 16k | `▁ж , ▁ж ▁— ▁кириллтәи ▁аҩыратә ▁нбан . ▁азхьарԥшқәа ▁graphemica ... (+7 more)` | 17 |
+| 32k | `▁ж , ▁ж ▁— ▁кириллтәи ▁аҩыратә ▁нбан . ▁азхьарԥшқәа ▁graphemica ... (+7 more)` | 17 |
+| 64k | `▁ж , ▁ж ▁— ▁кириллтәи ▁аҩыратә ▁нбан . ▁азхьарԥшқәа ▁graphemica ... (+7 more)` | 17 |
 
 
 ### Key Findings
 
 - **Best Compression:** 64k achieves 4.193x compression
-- **Lowest UNK Rate:** 8k with 0.1494% unknown tokens
+- **Lowest UNK Rate:** 8k with 0.1493% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 724 | 9.50 | 5,814 | 51.4% | 72.0% |
-| **2-gram** | Subword | 363 | 8.50 | 4,104 | 60.3% | 96.8% |
-| **3-gram** | Word | 252 🏆 | 7.98 | 5,216 | 66.5% | 80.6% |
-| **3-gram** | Subword | 2,675 | 11.39 | 28,199 | 28.1% | 67.5% |
-| **4-gram** | Word | 342 | 8.42 | 9,800 | 64.0% | 74.0% |
-| **4-gram** | Subword | 11,090 | 13.44 | 112,541 | 16.8% | 44.7% |
+| **2-gram** | Word | 723 | 9.50 | 5,814 | 51.5% | 72.0% |
+| **2-gram** | Subword | 363 | 8.51 | 4,117 | 60.3% | 96.8% |
+| **3-gram** | Word | 252 | 7.98 | 5,218 | 66.6% | 80.6% |
+| **3-gram** | Subword | 2,678 | 11.39 | 28,284 | 28.1% | 67.5% |
+| **4-gram** | Word | 341 | 8.41 | 9,794 | 64.0% | 74.0% |
+| **4-gram** | Subword | 11,104 | 13.44 | 112,814 | 16.8% | 44.7% |
+| **5-gram** | Word | 198 🏆 | 7.63 | 7,301 | 69.5% | 78.6% |
+| **5-gram** | Subword | 26,131 | 14.67 | 211,528 | 13.8% | 34.5% |
 
 ### Top 5 N-grams by Size
 
@@ -154,7 +166,7 @@ Below are sample sentences tokenized with each vocabulary size:
 | 2 | `иит рыԥсҭазаара` | 3,938 |
 | 3 | `рашәарамза ԥхынгәымза` | 3,603 |
 | 4 | `жәабранмза хәажәкырамза` | 3,603 |
-| 5 | `ажьырныҳәамза жәабранмза` | 3,602 |
+| 5 | `цәыббрамза жьҭаарамза` | 3,602 |
 
 **3-grams (Word):**
 
@@ -163,55 +175,75 @@ Below are sample sentences tokenized with each vocabulary size:
 | 1 | `иит рыԥсҭазаара иалҵит` | 3,938 |
 | 2 | `цәыббрамза жьҭаарамза абҵарамза` | 3,602 |
 | 3 | `нанҳәамза цәыббрамза жьҭаарамза` | 3,601 |
-| 4 | `рашәарамза ԥхынгәымза нанҳәамза` | 3,601 |
-| 5 | `мшаԥымза лаҵарамза рашәарамза` | 3,601 |
+| 4 | `жьҭаарамза абҵарамза ԥхынҷкәынмза` | 3,601 |
+| 5 | `лаҵарамза рашәарамза ԥхынгәымза` | 3,601 |
 
 **4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `мшаԥымза лаҵарамза рашәарамза ԥхынгәымза` | 3,601 |
-| 2 | `лаҵарамза рашәарамза ԥхынгәымза нанҳәамза` | 3,601 |
-| 3 | `рашәарамза ԥхынгәымза нанҳәамза цәыббрамза` | 3,601 |
-| 4 | `ԥхынгәымза нанҳәамза цәыббрамза жьҭаарамза` | 3,601 |
-| 5 | `нанҳәамза цәыббрамза жьҭаарамза абҵарамза` | 3,601 |
+| 1 | `цәыббрамза жьҭаарамза абҵарамза ԥхынҷкәынмза` | 3,601 |
+| 2 | `нанҳәамза цәыббрамза жьҭаарамза абҵарамза` | 3,601 |
+| 3 | `ԥхынгәымза нанҳәамза цәыббрамза жьҭаарамза` | 3,601 |
+| 4 | `мшаԥымза лаҵарамза рашәарамза ԥхынгәымза` | 3,601 |
+| 5 | `лаҵарамза рашәарамза ԥхынгәымза нанҳәамза` | 3,601 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `мшаԥымза лаҵарамза рашәарамза ԥхынгәымза нанҳәамза` | 3,601 |
+| 2 | `рашәарамза ԥхынгәымза нанҳәамза цәыббрамза жьҭаарамза` | 3,601 |
+| 3 | `ԥхынгәымза нанҳәамза цәыббрамза жьҭаарамза абҵарамза` | 3,601 |
+| 4 | `нанҳәамза цәыббрамза жьҭаарамза абҵарамза ԥхынҷкәынмза` | 3,601 |
+| 5 | `лаҵарамза рашәарамза ԥхынгәымза нанҳәамза цәыббрамза` | 3,601 |
 
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `а _` | 154,707 |
-| 2 | `_ а` | 149,768 |
-| 3 | `р а` | 100,539 |
-| 4 | `а р` | 84,594 |
-| 5 | `ә а` | 75,990 |
+| 1 | `а _` | 154,936 |
+| 2 | `_ а` | 150,057 |
+| 3 | `р а` | 100,657 |
+| 4 | `а р` | 84,729 |
+| 5 | `ә а` | 76,114 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `а р а` | 50,267 |
-| 2 | `м з а` | 45,869 |
-| 3 | `з а _` | 44,868 |
-| 4 | `а _ а` | 35,436 |
-| 5 | `а м з` | 31,355 |
+| 1 | `а р а` | 50,339 |
+| 2 | `м з а` | 45,875 |
+| 3 | `з а _` | 44,872 |
+| 4 | `а _ а` | 35,534 |
+| 5 | `а м з` | 31,361 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `м з а _` | 44,436 |
-| 2 | `а м з а` | 30,784 |
-| 3 | `р а м з` | 22,744 |
-| 4 | `а р а _` | 19,480 |
-| 5 | `қ ә а _` | 17,506 |
+| 1 | `м з а _` | 44,438 |
+| 2 | `а м з а` | 30,790 |
+| 3 | `р а м з` | 22,745 |
+| 4 | `а р а _` | 19,530 |
+| 5 | `қ ә а _` | 17,562 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `а м з а _` | 29,604 |
+| 2 | `р а м з а` | 22,366 |
+| 3 | `а р а м з` | 15,138 |
+| 4 | `т ә и _ а` | 11,926 |
+| 5 | `а қ ә а _` | 9,350 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 3-gram (word) with 252
+- **Best Perplexity:** 5-gram (word) with 198
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~45% of corpus
+- **Coverage:** Top-1000 patterns cover ~34% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.6658 | 1.586 | 3.61 | 90,583 | 33.4% |
-| **1** | Subword | 1.3373 | 2.527 | 10.82 | 872 | 0.0% |
-| **2** | Word | 0.1207 | 1.087 | 1.22 | 326,705 | 87.9% |
-| **2** | Subword | 1.0105 | 2.015 | 5.95 | 9,435 | 0.0% |
-| **3** | Word | 0.0295 | 1.021 | 1.04 | 396,742 | 97.1% |
-| **3** | Subword | 0.7768 | 1.713 | 3.69 | 56,063 | 22.3% |
-| **4** | Word | 0.0100 🏆 | 1.007 | 1.01 | 412,289 | 99.0% |
-| **4** | Subword | 0.5289 | 1.443 | 2.33 | 206,852 | 47.1% |
+| **1** | Word | 0.6658 | 1.586 | 3.61 | 90,782 | 33.4% |
+| **1** | Subword | 1.3353 | 2.523 | 10.79 | 879 | 0.0% |
+| **2** | Word | 0.1206 | 1.087 | 1.22 | 327,437 | 87.9% |
+| **2** | Subword | 1.0094 | 2.013 | 5.94 | 9,477 | 0.0% |
+| **3** | Word | 0.0294 | 1.021 | 1.04 | 397,532 | 97.1% |
+| **3** | Subword | 0.7766 | 1.713 | 3.69 | 56,288 | 22.3% |
+| **4** | Word | 0.0100 🏆 | 1.007 | 1.01 | 413,065 | 99.0% |
+| **4** | Subword | 0.5281 | 1.442 | 2.33 | 207,598 | 47.2% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `уи иалнаршеит абанкбжьаратә ҳасабкрақәа рҭыԥ инаркны шықәсанӡа наринџьованиархангельский а лазарев м...`
-2. `рыԥсҭазаара иалҵит шықәса нанҳәа 5 ԥхынҷкәы́н 22 27 азы иҭыҵит раԥхьа ақырҭуа ҭыԥҳа викторина ж ж`
-3. `иит рыԥсҭазаара иалҵит дидим халкентер абырзен бызшәала адраматургиа иақәшәаны ииашаҵәҟьаны еилкааӡа...`
+1. `уи зыхҟьаз зеиҧш дыҟамыз аҧҳәызба ссир иргылеит еидҵоу қырҭтәыла адемократиатә хдырра асоциалтә хьча...`
+2. `рыԥсҭазаара иалҵит пиотр актәи амаӡаныҟәгаҩыс ш вуковар vukovar jedna prica ш азхьарԥшқәа heritagesi...`
+3. `иит рыԥсҭазаара иалҵит кринагор абырзен бызшәа афранцыз италиа иалаигалоит флоренцианӡагьы инеиуеит ...`
 
 **Context Size 2:**
 
-1. `иит рыԥсҭазаара иалҵит клеопатра селена ii марк антонии клеопатреи antony and cleopatra кориолан cor...`
-2. `рашәарамза ԥхынгәымза нанҳәамза цәыббрамза жьҭаарамза абҵарамза ԥхынҷкәынмза иит рыԥсҭазаара иалҵит ...`
-3. `жәабранмза хәажәкырамза мшаԥымза лаҵарамза рашәарамза ԥхынгәымза нанҳәамза цәыббрамза жьҭаарамза абҵ...`
+1. `иит рыԥсҭазаара иалҵит октавиан август аԥеиԥа диит ҳ ҟ 326 мцхеҭа ҳ ҟ 14 ш абанктә система`
+2. `жәабранмза хәажәкырамза мшаԥымза лаҵарамза рашәарамза ԥхынгәымза нанҳәамза цәыббрамза жьҭаарамза абҵ...`
+3. `рашәарамза ԥхынгәымза нанҳәамза цәыббрамза жьҭаарамза абҵарамза ԥхынҷкәынмза иит рыԥсҭазаара иалҵит ...`
 
 **Context Size 3:**
 
-1. `цәыббрамза жьҭаарамза абҵарамза ԥхынҷкәынмза иит калигула римтәи аимператор дыԥсит 69 ш рыԥсҭазаара ...`
+1. `цәыббрамза жьҭаарамза абҵарамза ԥхынҷкәынмза иит рыԥсҭазаара иалҵит аныҳәақәа араԥтә ар амш аҳәаахьч...`
 2. `жьҭаарамза абҵарамза ԥхынҷкәынмза иит рыԥсҭазаара иалҵит аныҳәақәа араԥтә ар амш аҳәаахьчаҩцәа рамш ...`
 3. `ажьырныҳәамза жәабранмза хәажәкырамза мшаԥымза лаҵарамза рашәарамза ԥхынгәымза нанҳәамза цәыббрамза ...`
 
 **Context Size 4:**
 
-1. `нанҳәамза цәыббрамза жьҭаарамза абҵарамза ԥхынҷкәынмза иит рыԥсҭазаара иалҵит арминии германиатә хер...`
-2. `цәыббрамза жьҭаарамза абҵарамза ԥхынҷкәынмза иит рыԥсҭазаара иалҵит арминии германиатә херуски аимшь...`
-3. `рашәарамза ԥхынгәымза нанҳәамза цәыббрамза жьҭаарамза абҵарамза ԥхынҷкәынмза иит рыԥсҭазаара иалҵит ...`
+1. `цәыббрамза жьҭаарамза абҵарамза ԥхынҷкәынмза иит рыԥсҭазаара иалҵит клавдиа пульхра римтәи аамсҭаԥхә...`
+2. `лаҵарамза рашәарамза ԥхынгәымза нанҳәамза цәыббрамза жьҭаарамза абҵарамза ԥхынҷкәынмза иит рыԥсҭазаа...`
+3. `ԥхынгәымза нанҳәамза цәыббрамза жьҭаарамза абҵарамза ԥхынҷкәынмза иит рыԥсҭазаара иалҵит клавдиа пул...`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `аил_даит.._аарԥа`
-2. `_дигьҭақәанбрббс`
-3. `иунмақә_ser_иала`
+1. `аякарамаҟәаҿы_«п`
+2. `_жьы_ажәынқәсп_и`
+3. `иха_аббарран._ло`
 
 **Context Size 2:**
 
-1. `а_иазара,_иа_мамҭ`
-2. `_ашықәаԥсҭақәа_из`
-3. `рамза_иционва,_30`
+1. `а_уи_ахьы_иркую_с`
+2. `_ареит._ара_ихьам`
+3. `рала_ԥхын,_хьшара`
 
 **Context Size 3:**
 
-1. `ара_амшын._уи_аҩны`
-2. `мза_ԥхынгәымза_абе`
-3. `за_мшаԥысуа_кәу._а`
+1. `араҟнытә_бызшәалеи`
+2. `мза_жьҭаарамза_жәа`
+3. `за_ԥхынгәырый_фано`
 
 **Context Size 4:**
 
-1. `мза_нанҳәамҭа._аҩаӡ`
-2. `амза_рашәара_мап_рц`
-3. `рамза_ԥхынгәы_9,_ш.`
+1. `мза_ракәзар,_зныз_х`
+2. `амза_рашәара,_шықәс`
+3. `рамза_ԥхынгәымза_ла`
 
 
 ### Key Findings
 
 - **Best Predictability:** Context-4 (word) with 99.0% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (206,852 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (207,598 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,23 +346,23 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 32,686 |
-| Total Tokens | 440,475 |
-| Mean Frequency | 13.48 |
+| Vocabulary Size | 32,744 |
+| Total Tokens | 441,086 |
+| Mean Frequency | 13.47 |
 | Median Frequency | 3 |
-| Frequency Std Dev | 100.81 |
+| Frequency Std Dev | 100.78 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | уи | 4,159 |
+| 1 | уи | 4,161 |
 | 2 | рыԥсҭазаара | 4,025 |
 | 3 | иит | 3,987 |
 | 4 | иалҵит | 3,980 |
 | 5 | лаҵарамза | 3,752 |
 | 6 | жәабранмза | 3,722 |
-| 7 | хәажәкырамза | 3,701 |
+| 7 | хәажәкырамза | 3,702 |
 | 8 | абҵарамза | 3,701 |
 | 9 | нанҳәамза | 3,696 |
 | 10 | ԥхынҷкәынмза | 3,696 |
@@ -339,23 +371,23 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | slang | 2 |
-| 2 | пуэрто | 2 |
-| 3 | испантәи | 2 |
-| 4 | reggaetón | 2 |
-| 5 | маратон | 2 |
-| 6 | урымтәыла | 2 |
-| 7 | византии | 2 |
-| 8 | белобров | 2 |
-| 9 | длины | 2 |
-| 10 | акармара | 2 |
+| 1 | sons | 2 |
+| 2 | extended | 2 |
+| 3 | stream | 2 |
+| 4 | block | 2 |
+| 5 | stru | 2 |
+| 6 | compressed | 2 |
+| 7 | deflate | 2 |
+| 8 | january | 2 |
+| 9 | видеохәмарроуп | 2 |
+| 10 | роблокс | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 0.9638 |
-| R² (Goodness of Fit) | 0.995375 |
+| Zipf Coefficient | 0.9626 |
+| R² (Goodness of Fit) | 0.995444 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
@@ -363,7 +395,7 @@ Below are text samples generated from each subword-based Markov chain model:
 | Top N Words | Coverage |
 |-------------|----------|
 | Top 100 | 30.3% |
-| Top 1,000 | 55.8% |
+| Top 1,000 | 55.7% |
 | Top 5,000 | 76.9% |
 | Top 10,000 | 85.7% |
 
@@ -371,7 +403,7 @@ Below are text samples generated from each subword-based Markov chain model:
 
 - **Zipf Compliance:** R²=0.9954 indicates excellent adherence to Zipf's law
 - **High Frequency Dominance:** Top 100 words cover 30.3% of corpus
-- **Long Tail:** 22,686 words needed for remaining 14.3% coverage
+- **Long Tail:** 22,744 words needed for remaining 14.3% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.8185 🏆 | 0.3505 | N/A | N/A |
-| **mono_64d** | 64 | 0.6063 | 0.2895 | N/A | N/A |
-| **mono_128d** | 128 | 0.1746 | 0.2841 | N/A | N/A |
+| **mono_32d** | 32 | 0.8394 | 0.3485 | N/A | N/A |
+| **mono_64d** | 64 | 0.5679 | 0.2942 | N/A | N/A |
+| **mono_128d** | 128 | 0.1636 | 0.2836 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8394 🏆 | 0.3421 | 0.0220 | 0.1360 |
+| **aligned_64d** | 64 | 0.5679 | 0.2946 | 0.0360 | 0.1960 |
+| **aligned_128d** | 128 | 0.1636 | 0.2850 | 0.0420 | 0.2180 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.8185 (more uniform distribution)
+- **Best Isotropy:** aligned_32d with 0.8394 (more uniform distribution)
 - **Semantic Density:** Average pairwise similarity of 0.3080. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Alignment Quality:** Aligned models achieve up to 4.2% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **2.615** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **1.280** | High formulaic/idiomatic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -426,19 +461,19 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Prefixes
 | Prefix | Examples |
 |--------|----------|
-| `-иа` | иаанхон, иаҳәшьа, иагьлыхьӡиҵоит |
+| `-иа` | иалаҵоу, иааргазар, иадлоит |
 
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-а` | азербайджана, рыҟамлара, аҭӡысахьақәа |
-| `-әа` | аҭӡысахьақәа, льготақәа, ааимҭақәа |
-| `-қәа` | аҭӡысахьақәа, льготақәа, ааимҭақәа |
-| `-ит` | еиҭарҳәоит, иагьлыхьӡиҵоит, рыдиулоит |
-| `-ра` | рыҟамлара, архынҳәра, ахынҳәра |
-| `-еи` | аԥсреи, русқәеи, аглобализациеи |
-| `-тә` | антилопатә, аҳазылхратә, апрозатә |
-| `-ақәа` | аҭӡысахьақәа, льготақәа, ааимҭақәа |
+| `-а` | акандидатцәа, азура, ашәара |
+| `-әа` | акандидатцәа, акрақәа, аконсультациақәа |
+| `-ит` | иҟамлеит, дагәыланахалоит, дашьҭалоит |
+| `-қәа` | акрақәа, аконсультациақәа, дунеихәаԥшрақәа |
+| `-ра` | азура, ашәара, рықәцара |
+| `-тә` | алашаратә, аҵакырадгьылтә, аетнографиатә |
+| `-еи` | ргәыԥқәеи, астатуиақәеи, аизгақәеи |
+| `-еит` | иҟамлеит, ԥхасҭахеит, игәарҭеит |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -446,18 +481,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `гыла` | 1.69x | 82 contexts | гылан, ргылан, игылаз |
-| `аҵар` | 1.72x | 38 contexts | аҵара, лаҵара, хаҵара |
-| `әыла` | 1.72x | 34 contexts | тәыла, атәыла, тәылан |
-| `ықәс` | 1.85x | 26 contexts | шықәс, шықәсы, шықәса |
-| `әара` | 1.47x | 58 contexts | шәара, акәара, аҟәара |
-| `арам` | 2.03x | 17 contexts | нарам, харам, арамка |
-| `қәса` | 2.02x | 16 contexts | шықәса, щықәса, жәшықәса |
-| `шәар` | 1.69x | 26 contexts | шәара, ршәарц, шәарах |
-| `ҭаза` | 2.47x | 8 contexts | иԥсҭазара, ԥсҭазаара, ипсҭазаара |
+| `гыла` | 1.73x | 82 contexts | гылан, ргылан, дгылан |
+| `ықәс` | 1.84x | 26 contexts | шықәс, щықәса, ашықәс |
+| `әыла` | 1.68x | 34 contexts | тәыла, тәылак, ртәыла |
+| `аҵар` | 1.63x | 38 contexts | аҵара, лаҵара, аҵареи |
+| `қәса` | 1.96x | 16 contexts | щықәса, шықәса, шиқәсазы |
+| `арам` | 1.86x | 17 contexts | харам, нарам, гуарам |
 | `азаа` | 1.69x | 23 contexts | лазаа, амазаап, иазааит |
-| `ыҳәа` | 1.62x | 22 contexts | ныҳәа, иныҳәа, ныҳәан |
-| `заар` | 2.08x | 10 contexts | акзаара, акзаарак, ракзаара |
+| `әара` | 1.30x | 58 contexts | шәара, акәара, ҿҳәара |
+| `ҭаза` | 2.37x | 8 contexts | иԥсҭазара, ԥсҭазаара, иԥсҭазаара |
+| `шәар` | 1.56x | 26 contexts | шәара, шәарах, ашәара |
+| `заар` | 2.09x | 10 contexts | акзаара, аҟазаара, акзаареи |
+| `ыҳәа` | 1.57x | 22 contexts | ныҳәа, рныҳәа, иныҳәа |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -465,16 +500,15 @@ This table shows which prefixes and suffixes most frequently co-occur on the sam
 
 | Prefix | Suffix | Frequency | Examples |
 |--------|--------|-----------|----------|
-| `-иа` | `-ит` | 97 words | иахьӡырҵеит, иаҭаауеит |
-| `-иа` | `-еит` | 54 words | иахьӡырҵеит, иаҭаауеит |
-| `-иа` | `-а` | 42 words | иалихуа, иадыруа |
-| `-иа` | `-әа` | 14 words | иаламҵакәа, иажәа |
-| `-иа` | `-ра` | 5 words | иабшьҭра, иааӡара |
-| `-иа` | `-тә` | 5 words | иаҳратә, иашахаҵаратә |
-| `-иа` | `-қәа` | 5 words | иашақәа, ианҵамҭақәа |
-| `-иа` | `-ақәа` | 4 words | иашақәа, ианҵамҭақәа |
-| `-иа` | `-еи` | 3 words | иашьцәеи, иамхреи |
-| `-иа` | `-әи` | 1 words | иапониатәи |
+| `-иа` | `-ит` | 83 words | иаабоит, иацхраауеит |
+| `-иа` | `-еит` | 50 words | иацхраауеит, иартәеит |
+| `-иа` | `-а` | 43 words | ианырба, ианрылага |
+| `-иа` | `-әа` | 11 words | иацәыхарамкәа, иаламлакәа |
+| `-иа` | `-тә` | 5 words | иааникыларатә, иавтобиографиатә |
+| `-иа` | `-ра` | 3 words | иавторра, иамхра |
+| `-иа` | `-еи` | 2 words | ианԥсеи, иашьцәеи |
+| `-иа` | `-қәа` | 2 words | иажәақәа, иажәамаанақәа |
+| `-иа` | `-ақәа` | 1 words | иажәақәа, иажәамаанақәа |
 
 ### 6.5 Recursive Morpheme Segmentation
 
@@ -482,26 +516,28 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
+| анхарҭатә | **`анхарҭа-тә`** | 4.5 | `анхарҭа` |
+| рхыԥхьаӡараҟнытә | **`рхыԥхьаӡараҟны-тә`** | 4.5 | `рхыԥхьаӡараҟны` |
+| аӡхықәқәа | **`аӡхықә-қәа`** | 4.5 | `аӡхықә` |
+| астуденттә | **`астудент-тә`** | 4.5 | `астудент` |
 | аҳәынҭқарқәа | **`аҳәынҭқар-қәа`** | 4.5 | `аҳәынҭқар` |
-| мраҭашәаратә | **`мраҭаш-әа-ра-тә`** | 4.5 | `мраҭаш` |
-| ашьауӷатә | **`ашьауӷа-тә`** | 4.5 | `ашьауӷа` |
-| алахәылара | **`алахәыла-ра`** | 4.5 | `алахәыла` |
-| амитингқәа | **`амитинг-қәа`** | 4.5 | `амитинг` |
-| аметаллургиатә | **`аметаллургиа-тә`** | 4.5 | `аметаллургиа` |
-| рекологиатә | **`рекологиа-тә`** | 4.5 | `рекологиа` |
-| асиужетқәа | **`асиужет-қәа`** | 4.5 | `асиужет` |
-| аконцерттә | **`аконцерт-тә`** | 4.5 | `аконцерт` |
-| ипартиатә | **`ипартиа-тә`** | 4.5 | `ипартиа` |
-| афасадқәа | **`афасад-қәа`** | 4.5 | `афасад` |
-| астадионқәа | **`астадион-қәа`** | 4.5 | `астадион` |
-| аномерқәа | **`аномер-қәа`** | 4.5 | `аномер` |
-| аныҳәаратә | **`аныҳ-әа-ра-тә`** | 4.5 | `аныҳ` |
-| атерминқәа | **`атермин-қәа`** | 4.5 | `атермин` |
+| каталониатә | **`каталониа-тә`** | 4.5 | `каталониа` |
+| абиблиографиатә | **`абиблиографиа-тә`** | 4.5 | `абиблиографиа` |
+| аредакциатә | **`аредакциа-тә`** | 4.5 | `аредакциа` |
+| амилициатә | **`амилициа-тә`** | 4.5 | `амилициа` |
+| амилаҭқәа | **`амилаҭ-қәа`** | 4.5 | `амилаҭ` |
+| аекологиатә | **`аекологиа-тә`** | 4.5 | `аекологиа` |
+| адемографиатә | **`адемографиа-тә`** | 4.5 | `адемографиа` |
+| аконсервациатә | **`аконсервациа-тә`** | 4.5 | `аконсервациа` |
+| ауаҩытәыҩсатә | **`ауаҩытәыҩса-тә`** | 4.5 | `ауаҩытәыҩса` |
+| аелементқәа | **`аелемент-қәа`** | 4.5 | `аелемент` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language AB appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Abkhazian shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
+
+> **Note on Idiomaticity:** The high Idiomaticity Gap suggests a large number of frequent multi-word expressions or formulaic sequences that are statistically distinct from their component parts.
 
 ---
 ## 7. Summary & Recommendations
@@ -513,7 +549,7 @@ The language AB appears to be more isolating or has a highly fixed vocabulary. W
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
 | Tokenizer | **64k BPE** | Best compression (4.19x) |
-| N-gram | **3-gram** | Lowest perplexity (252) |
+| N-gram | **5-gram** | Lowest perplexity (198) |
 | Markov | **Context-4** | Highest predictability (99.0%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
@@ -728,4 +764,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 05:05:55*
+*Report Date: 2026-01-03 16:16:58*

models/embeddings/aligned/ab_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71c42722da6595336ced095cc89ab98dce46aa2764b97c56a059a072aa38c40f
+size 1036432309

models/embeddings/aligned/ab_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "ab", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/ab_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6554a5b2123562574512b96e8803301b6bd19ee23ecae768b30b02038949093
+size 65664

models/embeddings/aligned/ab_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "ab",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 927,
+  "vocab_size": 11860
+}

models/embeddings/aligned/ab_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26e75c47cb2d2d69347eb0db2b7206527bdfe9a74fd1254d6e81ebb46b8e86f4
+size 259323829

models/embeddings/aligned/ab_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "ab", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/ab_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61563ea00eba259124ac0cd787d4e2f7845d51497b14dbe427b37e79e0366086
+size 4224

models/embeddings/aligned/ab_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "ab",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 927,
+  "vocab_size": 11860
+}

models/embeddings/aligned/ab_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24b083b8f182ed5799370c8e2ae70954e4785710342ad0ebd087e7b12310f7f4
+size 518359989

models/embeddings/aligned/ab_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "ab", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/ab_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf87c42f43dc72a3c48c49eb5a683b1a5c25149e7ab05d83bee2dba66444b19b
+size 16512

models/embeddings/aligned/ab_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "ab",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 927,
+  "vocab_size": 11860
+}

models/embeddings/monolingual/ab_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:974accbf5e248437084213db34906d889d1387be4ba287f5bf3decc278094a76
-size 1036399852
+oid sha256:71c42722da6595336ced095cc89ab98dce46aa2764b97c56a059a072aa38c40f
+size 1036432309

models/embeddings/monolingual/ab_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 11829
+  "vocab_size": 11860
 }

models/embeddings/monolingual/ab_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:20cc29bc8688764f66dd79168f5b38b9bc2c37d4cff74832ca2676787afff53e
-size 259315180
+oid sha256:26e75c47cb2d2d69347eb0db2b7206527bdfe9a74fd1254d6e81ebb46b8e86f4
+size 259323829

models/embeddings/monolingual/ab_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 11829
+  "vocab_size": 11860
 }

models/embeddings/monolingual/ab_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6e336f851f9509b219587c555a83bc96b5e5f8af5ceca5f7c676a4c11e0eec7
-size 518343404
+oid sha256:24b083b8f182ed5799370c8e2ae70954e4785710342ad0ebd087e7b12310f7f4
+size 518359989

models/embeddings/monolingual/ab_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 11829
+  "vocab_size": 11860
 }

models/subword_markov/ab_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:08d81996caede4f16305dff37bfaf00cc4aa080b329a05b6e813e43b32d4b5b1
-size 74300
+oid sha256:19cd311913da52a816b4226234ba5047af2e1d508ac9c65583ea9a88645432ac
+size 74098

models/subword_markov/ab_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "ab",
-  "unique_contexts": 872,
-  "total_transitions": 4086915
+  "unique_contexts": 879,
+  "total_transitions": 4093303
 }

models/subword_markov/ab_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:391adb8ed94471fe378fd39fa4bafc27bc22517c04a0688782555f658bfa5660
-size 430405
+oid sha256:927b982ad94c14844b6d7630912a44f97b2ed837ecc6612f0334f1f20f72c9e3
+size 431394

models/subword_markov/ab_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "ab",
-  "unique_contexts": 9435,
-  "total_transitions": 4080892
+  "unique_contexts": 9477,
+  "total_transitions": 4087276
 }

models/subword_markov/ab_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2554570f44dad84400a3a91e792ec596f90a065c3a673af002a820c70dca44e9
-size 1518240
+oid sha256:7f54ce1a72b48d81d104298015c3c2deb43f8272a78af7071406a6e3a7b786f0
+size 1525586

models/subword_markov/ab_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "ab",
-  "unique_contexts": 56063,
-  "total_transitions": 4074869
+  "unique_contexts": 56288,
+  "total_transitions": 4081249
 }

models/subword_markov/ab_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a36f4483bb607e4182b1e8a367258cf1570a36ccf6debdfa0c5ed6d83eb6baa9
-size 4368269
+oid sha256:96368e0437f26828beb02a17e213d6db03b94d43cfc0d2d0a1f95b152a4198e8
+size 4373446

models/subword_markov/ab_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "ab",
-  "unique_contexts": 206852,
-  "total_transitions": 4068846
+  "unique_contexts": 207598,
+  "total_transitions": 4075222
 }

models/subword_ngram/ab_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6546e93ab975088eb711841de0aac0eef95cedd64a0648eef8735695ed55ff88
-size 54917
+oid sha256:0a1ed758e72646135180f530b31bbda049e7ce468f83f1559ec140a8ce2d91db
+size 54947

models/subword_ngram/ab_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "ab",
-  "unique_ngrams": 4104,
-  "total_ngrams": 4086915
+  "unique_ngrams": 4117,
+  "total_ngrams": 4093303
 }

models/subword_ngram/ab_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7b1b02523b1b148c8a70500352648cf3a5210acb9cff8bd3b8c613fbe55e72ef
-size 368926
+oid sha256:25e278dc8f9634827e842fd4fa0a7ab1634032e48a71621686047e4cd1ca2c00
+size 369086

models/subword_ngram/ab_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "ab",
-  "unique_ngrams": 28199,
-  "total_ngrams": 4080892
+  "unique_ngrams": 28284,
+  "total_ngrams": 4087276
 }

models/subword_ngram/ab_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7eebb21889eb39845863d08c2bb3db59acb306cc610d9cff31719d1fccb8a9f
-size 1420721
+oid sha256:665986334ceeda8aa48fbc99ac922f66ba2fa5fe34df93b4d6ba07fc1f9d5d48
+size 1427310

models/subword_ngram/ab_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "ab",
-  "unique_ngrams": 112541,
-  "total_ngrams": 4074869
+  "unique_ngrams": 112814,
+  "total_ngrams": 4081249
 }

models/subword_ngram/ab_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:839b1b7f710bb721be0571bcffc69c450506f2c7de4a3cede70a0d8f89d9a2c5
+size 2860189

models/subword_ngram/ab_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "ab",
+  "unique_ngrams": 211528,
+  "total_ngrams": 4075222
+}

models/tokenizer/ab_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a2085566fc5290d88287b8216db2b990d022c905f7619c834e7a919406ca669
-size 569009
+oid sha256:faab02fa0c6d322c4b29526ae72183b3c6bdefcdd1dde3661ce29441818b789e
+size 569291

models/tokenizer/ab_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1e4a3fa167aa62fd56d1a517f1285f051a7587310895cdc82a80895e1bd8badc
-size 936940
+oid sha256:6373f1b123f84fb1dc1394e345603a0a661907f056224dd29e84248278023e8e
+size 936901

models/tokenizer/ab_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09581380da9729bf63f3885bf0e54ee06e9b0659cd517ad67eab83cf01150542
-size 1690301
+oid sha256:33980fec6cedaeeab4a3b516f2447c745616b229192936ac6f9bcc71b8c92cd0
+size 1690475

models/tokenizer/ab_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb1da49fc5cbf61e37585d974f3bcb0262f7b8e403c201974781b65a1079fa28
-size 402830
+oid sha256:c621351e7846d8e21f2dc64c9034dd95cd0718a1712a0b7ed4b16694ad52bf38
+size 403151

models/vocabulary/ab_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b9954423f516ffad7b6b395410e0a3ed33a13656770c69fcd1e37b9a2500496
-size 613460
+oid sha256:96921566daf39bc2fc2c78e494582b420e246beb86cef2d2868dd655be330fad
+size 618434

models/vocabulary/ab_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "ab",
-  "vocabulary_size": 32686,
+  "vocabulary_size": 32744,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.18190225895758938,
+    "type_token_ratio": 0.18202523582212732,
     "coverage": {
-      "top_100": 0.26752598001845684,
-      "top_1000": 0.4927917987401196,
-      "top_5000": 0.6798940737471412,
-      "top_10000": 0.7575291899049071
+      "top_100": 0.26734987810765387,
+      "top_1000": 0.4923108665474118,
+      "top_5000": 0.679398090186314,
+      "top_10000": 0.7571407681321125
     },
-    "hapax_count": 57985,
-    "hapax_ratio": 0.639509876366203,
-    "total_documents": 6023
+    "hapax_count": 58125,
+    "hapax_ratio": 0.6396570887761502,
+    "total_documents": 6027
   }
 }

models/word_markov/ab_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:477c26b2510447d28ecaa37e0e7ab40a517773cfffb1210fbde670a44914bc00
-size 4536676
+oid sha256:2bf7260c71b0507ac13f5ca76a0540969736af005153e3ea30a842a8fbc49138
+size 4532845

models/word_markov/ab_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "ab",
-  "unique_contexts": 90583,
-  "total_transitions": 492437
+  "unique_contexts": 90782,
+  "total_transitions": 493184
 }

models/word_markov/ab_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4bc4862d6258993f1f724d0d5118dc5b0d53b173d1761b40efd9fab5f483159
-size 10110358
+oid sha256:a16ee7a61ea5b043c3495cdd5ddaadb1eaab15dfa5e1487cdeaa37c6f45c0fd0
+size 10123960

models/word_markov/ab_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "ab",
-  "unique_contexts": 326705,
-  "total_transitions": 486414
+  "unique_contexts": 327437,
+  "total_transitions": 487157
 }