Upload all models and assets for an (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: an
-language_name: AN
+language_name: Aragonese
 language_family: romance_iberian
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-romance_iberian
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.267
+    value: 4.275
   - name: best_isotropy
     type: isotropy
-    value: 0.8193
+    value: 0.8230
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# AN - Wikilangs Models
+# Aragonese - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **AN** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Aragonese** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.551x | 3.55 | 0.1259% | 1,215,405 |
-| **16k** | 3.845x | 3.85 | 0.1363% | 1,122,403 |
-| **32k** | 4.084x | 4.08 | 0.1448% | 1,056,930 |
-| **64k** | 4.267x 🏆 | 4.27 | 0.1513% | 1,011,533 |
+| **8k** | 3.559x | 3.56 | 0.1247% | 1,207,427 |
+| **16k** | 3.854x | 3.85 | 0.1351% | 1,114,964 |
+| **32k** | 4.092x | 4.09 | 0.1434% | 1,050,138 |
+| **64k** | 4.275x 🏆 | 4.28 | 0.1498% | 1,005,070 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Anyos: - - Decenios: Anyos - Anyos - Anyos Sieglos: Sieglo X - Sieglo XI - Siegl...`
+**Sample 1:** `CA Monzón puet estar: O Centro Atlético Monzón. O Club de Fútbol Atlético de Mon...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁anyos : ▁- ▁- ▁decenios : ▁anyos ▁- ▁anyos ▁- ... (+15 more)` | 25 |
-| 16k | `▁anyos : ▁- ▁- ▁decenios : ▁anyos ▁- ▁anyos ▁- ... (+15 more)` | 25 |
-| 32k | `▁anyos : ▁- ▁- ▁decenios : ▁anyos ▁- ▁anyos ▁- ... (+15 more)` | 25 |
-| 64k | `▁anyos : ▁- ▁- ▁decenios : ▁anyos ▁- ▁anyos ▁- ... (+15 more)` | 25 |
+| 8k | `▁ca ▁monzón ▁puet ▁estar : ▁o ▁centro ▁at lético ▁monzón ... (+10 more)` | 20 |
+| 16k | `▁ca ▁monzón ▁puet ▁estar : ▁o ▁centro ▁atlético ▁monzón . ... (+8 more)` | 18 |
+| 32k | `▁ca ▁monzón ▁puet ▁estar : ▁o ▁centro ▁atlético ▁monzón . ... (+8 more)` | 18 |
+| 64k | `▁ca ▁monzón ▁puet ▁estar : ▁o ▁centro ▁atlético ▁monzón . ... (+8 more)` | 18 |
 
-**Sample 2:** `Lo Buçon (en francés Aubusson) ye una localidat y comuna francesa situada en o d...`
+**Sample 2:** `En ista lista s'incluyen toz os presidents d'o Real Zaragoza dica hue: María Gay...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁lo ▁bu ç on ▁( en ▁francés ▁a ub us ... (+28 more)` | 38 |
-| 16k | `▁lo ▁bu çon ▁( en ▁francés ▁a ub us son ... (+25 more)` | 35 |
-| 32k | `▁lo ▁bu çon ▁( en ▁francés ▁aub us son ) ... (+24 more)` | 34 |
-| 64k | `▁lo ▁bu çon ▁( en ▁francés ▁aub us son ) ... (+22 more)` | 32 |
+| 8k | `▁en ▁ista ▁lista ▁s ' incluy en ▁toz ▁os ▁presid ... (+24 more)` | 34 |
+| 16k | `▁en ▁ista ▁lista ▁s ' incluyen ▁toz ▁os ▁presidents ▁d ... (+22 more)` | 32 |
+| 32k | `▁en ▁ista ▁lista ▁s ' incluyen ▁toz ▁os ▁presidents ▁d ... (+22 more)` | 32 |
+| 64k | `▁en ▁ista ▁lista ▁s ' incluyen ▁toz ▁os ▁presidents ▁d ... (+22 more)` | 32 |
 
-**Sample 3:** `Holzmann ye un lugar d'o municipio de Chieming en o sud-este de Bavera, Alemanya...`
+**Sample 3:** `Roitwalchen ye un lugar d'o municipio de Traunstein en o sud-este de Bavera, Ale...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁hol z mann ▁ye ▁un ▁lugar ▁d ' o ▁municipio ... (+28 more)` | 38 |
-| 16k | `▁hol z mann ▁ye ▁un ▁lugar ▁d ' o ▁municipio ... (+28 more)` | 38 |
-| 32k | `▁holz mann ▁ye ▁un ▁lugar ▁d ' o ▁municipio ▁de ... (+26 more)` | 36 |
-| 64k | `▁holz mann ▁ye ▁un ▁lugar ▁d ' o ▁municipio ▁de ... (+26 more)` | 36 |
+| 8k | `▁ro it wal chen ▁ye ▁un ▁lugar ▁d ' o ... (+28 more)` | 38 |
+| 16k | `▁ro it wal chen ▁ye ▁un ▁lugar ▁d ' o ... (+28 more)` | 38 |
+| 32k | `▁ro it wal chen ▁ye ▁un ▁lugar ▁d ' o ... (+28 more)` | 38 |
+| 64k | `▁ro it walchen ▁ye ▁un ▁lugar ▁d ' o ▁municipio ... (+27 more)` | 37 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.267x compression
-- **Lowest UNK Rate:** 8k with 0.1259% unknown tokens
+- **Best Compression:** 64k achieves 4.275x compression
+- **Lowest UNK Rate:** 8k with 0.1247% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 25,531 | 14.64 | 231,538 | 16.8% | 37.4% |
-| **2-gram** | Subword | 258 🏆 | 8.01 | 7,000 | 68.7% | 99.3% |
-| **3-gram** | Word | 86,752 | 16.40 | 456,958 | 8.4% | 23.0% |
-| **3-gram** | Subword | 2,153 | 11.07 | 52,839 | 25.8% | 73.4% |
-| **4-gram** | Word | 208,198 | 17.67 | 890,645 | 6.8% | 17.2% |
-| **4-gram** | Subword | 12,189 | 13.57 | 289,840 | 12.6% | 39.7% |
+| **2-gram** | Word | 25,712 | 14.65 | 233,669 | 16.7% | 37.4% |
+| **2-gram** | Subword | 257 🏆 | 8.01 | 7,000 | 68.7% | 99.3% |
+| **3-gram** | Word | 87,357 | 16.41 | 461,562 | 8.3% | 23.0% |
+| **3-gram** | Subword | 2,151 | 11.07 | 52,727 | 25.8% | 73.4% |
+| **4-gram** | Word | 209,676 | 17.68 | 900,576 | 6.8% | 17.2% |
+| **4-gram** | Subword | 12,170 | 13.57 | 289,768 | 12.6% | 39.7% |
+| **5-gram** | Word | 208,007 | 17.67 | 773,213 | 6.3% | 16.4% |
+| **5-gram** | Subword | 46,669 | 15.51 | 901,225 | 7.3% | 25.5% |
 
 ### Top 5 N-grams by Size
 
@@ -150,68 +162,88 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `d a` | 106,922 |
-| 2 | `d o` | 105,903 |
-| 3 | `en a` | 60,539 |
-| 4 | `en o` | 45,891 |
-| 5 | `de l` | 37,271 |
+| 1 | `d a` | 107,208 |
+| 2 | `d o` | 106,261 |
+| 3 | `en a` | 60,798 |
+| 4 | `en o` | 45,519 |
+| 5 | `de l` | 37,458 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `a provincia de` | 17,407 |
-| 2 | `d a provincia` | 13,389 |
-| 3 | `una superficie de` | 12,709 |
-| 4 | `suya población ye` | 12,409 |
-| 5 | `población ye de` | 12,354 |
+| 1 | `a provincia de` | 17,480 |
+| 2 | `d a provincia` | 13,447 |
+| 3 | `una superficie de` | 12,736 |
+| 4 | `suya población ye` | 12,405 |
+| 5 | `en una superficie` | 12,352 |
 
 **4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `suya población ye de` | 12,288 |
-| 2 | `en una superficie de` | 12,121 |
-| 3 | `d a provincia de` | 12,081 |
-| 4 | `habitants en una superficie` | 11,267 |
+| 1 | `suya población ye de` | 12,284 |
+| 2 | `en una superficie de` | 12,148 |
+| 3 | `d a provincia de` | 12,141 |
+| 4 | `habitants en una superficie` | 11,275 |
 | 5 | `a suya población ye` | 11,250 |
 
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `a suya población ye de` | 11,136 |
+| 2 | `habitants en una superficie de` | 11,095 |
+| 3 | `una densidat de población de` | 10,633 |
+| 4 | `km con una densidat de` | 7,736 |
+| 5 | `con una densidat de población` | 7,674 |
+
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `a _` | 1,856,533 |
-| 2 | `_ d` | 1,594,808 |
-| 3 | `e _` | 1,531,455 |
-| 4 | `s _` | 1,293,859 |
-| 5 | `n _` | 1,201,430 |
+| 1 | `a _` | 1,873,392 |
+| 2 | `_ d` | 1,605,638 |
+| 3 | `e _` | 1,544,207 |
+| 4 | `s _` | 1,309,585 |
+| 5 | `n _` | 1,215,896 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `_ d e` | 886,389 |
-| 2 | `d e _` | 768,800 |
-| 3 | `_ d '` | 488,988 |
-| 4 | `e n _` | 472,591 |
-| 5 | `_ e n` | 449,047 |
+| 1 | `_ d e` | 891,253 |
+| 2 | `d e _` | 772,067 |
+| 3 | `_ d '` | 491,537 |
+| 4 | `e n _` | 478,088 |
+| 5 | `_ e n` | 454,282 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `_ d e _` | 734,534 |
-| 2 | `_ e n _` | 392,856 |
-| 3 | `_ d ' a` | 233,826 |
-| 4 | `a _ d e` | 183,991 |
-| 5 | `_ c o n` | 176,849 |
+| 1 | `_ d e _` | 737,370 |
+| 2 | `_ e n _` | 397,348 |
+| 3 | `_ d ' a` | 234,868 |
+| 4 | `a _ d e` | 184,900 |
+| 5 | `_ c o n` | 179,093 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `a _ d e _` | 147,074 |
+| 2 | `_ q u e _` | 125,472 |
+| 3 | `c i ó n _` | 124,436 |
+| 4 | `o _ d e _` | 123,146 |
+| 5 | `_ d ' a _` | 106,742 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram (subword) with 258
+- **Best Perplexity:** 2-gram (subword) with 257
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~40% of corpus
+- **Coverage:** Top-1000 patterns cover ~25% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.9747 | 1.965 | 7.58 | 367,064 | 2.5% |
-| **1** | Subword | 0.7827 | 1.720 | 5.89 | 3,499 | 21.7% |
-| **2** | Word | 0.3412 | 1.267 | 2.01 | 2,775,765 | 65.9% |
-| **2** | Subword | 0.8316 | 1.780 | 5.33 | 20,583 | 16.8% |
-| **3** | Word | 0.1546 | 1.113 | 1.33 | 5,573,548 | 84.5% |
-| **3** | Subword | 0.7758 | 1.712 | 4.33 | 109,746 | 22.4% |
-| **4** | Word | 0.0738 🏆 | 1.052 | 1.14 | 7,417,165 | 92.6% |
-| **4** | Subword | 0.7143 | 1.641 | 3.37 | 474,426 | 28.6% |
+| **1** | Word | 0.9753 | 1.966 | 7.59 | 368,549 | 2.5% |
+| **1** | Subword | 0.7834 | 1.721 | 5.75 | 3,672 | 21.7% |
+| **2** | Word | 0.3415 | 1.267 | 2.01 | 2,791,626 | 65.9% |
+| **2** | Subword | 0.8176 | 1.763 | 5.23 | 21,123 | 18.2% |
+| **3** | Word | 0.1548 | 1.113 | 1.33 | 5,610,004 | 84.5% |
+| **3** | Subword | 0.7695 | 1.705 | 4.30 | 110,486 | 23.0% |
+| **4** | Word | 0.0739 🏆 | 1.053 | 1.14 | 7,469,366 | 92.6% |
+| **4** | Subword | 0.7129 | 1.639 | 3.37 | 474,961 | 28.7% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `de barcelona cuan obtiene un municipio espanyol o suyo segundo punto alchido d o brien moore`
-2. `d a z vs üü alto penedés información de chunio de argañán a camera a population`
-3. `a risilleta a suya identidat y bi ha una economía como dancing in europe bbc d`
+1. `de gúdar s j simpson cyril cusack georgi parvanov con a comarca d o mesmo significau`
+2. `d a fundación d o comité d abril linear 8 d ixe paisache pocino de marzo`
+3. `a circuscripción de linkedin pachina web autualment no la provincia de bixuteria fina cappa producti...`
 
 **Context Size 2:**
 
-1. `d a provincia de barcelona iste matrimonio naixoron 2 fillas a on a hansa montó as suyas`
-2. `d o far west anexionando muitos territorios que componeban a corona d aragón que explicitament denom...`
-3. `en a suya población ye de 643 habitants en una superficie de 16 01 491 teruel torralba`
+1. `d a provincia de beneviento 2 071 m o río xalón y provincia de zaragoza y atros`
+2. `d o poro nucleyar afi komˈple k so di ˈi ˈpoɾo nuˈkliar ye per propio cualsiquier craba`
+3. `en a pachina web oficial la procedencia d os mes aptos os caracters comuns con atras posesions`
 
 **Context Size 3:**
 
-1. `a provincia de uesca d as penyas de riglos solano n 42 27 29 21 e 0 27`
-2. `d a provincia de zaragoza en la suya part d o conchunto d o sud con a huerta`
-3. `una superficie de 88 4 km y una densidat de población de 10 44 hab km cifras oficiales`
+1. `a provincia de teruel comarca d a chacetania ye una d as prencipals actrices d o teatro y`
+2. `d a provincia de castellón de la plana alta la suya población ye de 651 habitants en germany`
+3. `una superficie de 62 99 km con una densidat de población de 8 hab km en ista localidat`
 
 **Context Size 4:**
 
-1. `suya población ye de en una superficie de 20 1 km y una densidat de población de 5 24`
-2. `en una superficie de 82 09 km con una densidat d hab km cheografía a localidat de gonnosnò ye`
-3. `d a provincia de chirona y partiu chudicial de teruel catálogo de pueblos y municipios de aragón est...`
+1. `suya población ye de 100 habitants en una superficie de 9 77 km con una densidat de población de`
+2. `en una superficie de 13 70 km con una densidat de población de 29 15 hab km cheografía a`
+3. `d a provincia de cordoba ta atros usos se veiga o caire desambigación o caire u simplamnet caire الق...`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `_ptonena_le_a_ev`
-2. `atarro_d'a_erra.`
-3. `e_ula_coril._qus`
+1. `_e_ra_ce_aren_pi`
+2. `al._iargo_rennye`
+3. `ebalobalo,_opo_c`
 
 **Context Size 2:**
 
-1. `a_forangacia_ingà`
-2. `_de_le_d'isten_co`
-3. `e_ye_suff_una_fes`
+1. `a_cominelde_al_de`
+2. `_d'a_s'esmarroixe`
+3. `e_au_gottorioními`
 
 **Context Size 3:**
 
-1. `_de_barrisor_intas`
-2. `de_a_prencias)._th`
-3. `_d'a_latín_a_lo_si`
+1. `_de_papezina_creye`
+2. `de_heyemios_poblac`
+3. `_d'africa_de_mayo»`
 
 **Context Size 4:**
 
-1. `_de_l'animent_y_pol`
-2. `_en_tiembre_de_davi`
-3. `_d'a_por_o_nuesta_y`
+1. `_de_l'anyos._ye_reg`
+2. `_en_la_menclaude._v`
+3. `_d'africantón_de_se`
 
 
 ### Key Findings
 
 - **Best Predictability:** Context-4 (word) with 92.6% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (474,426 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (474,961 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,64 +346,64 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 182,961 |
-| Total Tokens | 11,551,476 |
-| Mean Frequency | 63.14 |
+| Vocabulary Size | 183,928 |
+| Total Tokens | 11,661,736 |
+| Mean Frequency | 63.40 |
 | Median Frequency | 4 |
-| Frequency Std Dev | 2812.87 |
+| Frequency Std Dev | 2823.00 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | de | 738,428 |
-| 2 | d | 494,628 |
-| 3 | a | 437,762 |
-| 4 | en | 406,259 |
-| 5 | o | 301,745 |
-| 6 | y | 244,743 |
-| 7 | que | 126,444 |
-| 8 | l | 109,014 |
-| 9 | ye | 108,632 |
-| 10 | una | 104,144 |
+| 1 | de | 741,521 |
+| 2 | d | 497,145 |
+| 3 | a | 440,622 |
+| 4 | en | 410,893 |
+| 5 | o | 301,627 |
+| 6 | y | 247,568 |
+| 7 | que | 127,976 |
+| 8 | l | 109,848 |
+| 9 | ye | 109,774 |
+| 10 | una | 105,502 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | koftinoff | 2 |
-| 2 | landlocked | 2 |
-| 3 | hamidi | 2 |
-| 4 | tangy | 2 |
-| 5 | sélignac | 2 |
-| 6 | cômene | 2 |
-| 7 | varneton | 2 |
-| 8 | mackelway | 2 |
-| 9 | wigutow | 2 |
-| 10 | críspulo | 2 |
+| 1 | beljakova | 2 |
+| 2 | méchaly | 2 |
+| 3 | wiedemann | 2 |
+| 4 | limotte | 2 |
+| 5 | wlodkowski | 2 |
+| 6 | taos | 2 |
+| 7 | slovis | 2 |
+| 8 | samaha | 2 |
+| 9 | seros | 2 |
+| 10 | cookeville | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 1.0685 |
-| R² (Goodness of Fit) | 0.998276 |
+| Zipf Coefficient | 1.0690 |
+| R² (Goodness of Fit) | 0.998251 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 44.9% |
-| Top 1,000 | 66.9% |
+| Top 100 | 44.8% |
+| Top 1,000 | 66.8% |
 | Top 5,000 | 80.7% |
 | Top 10,000 | 85.9% |
 
 ### Key Findings
 
 - **Zipf Compliance:** R²=0.9983 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 44.9% of corpus
-- **Long Tail:** 172,961 words needed for remaining 14.1% coverage
+- **High Frequency Dominance:** Top 100 words cover 44.8% of corpus
+- **Long Tail:** 173,928 words needed for remaining 14.1% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.8177 | 0.3514 | N/A | N/A |
-| **mono_64d** | 64 | 0.8193 🏆 | 0.2716 | N/A | N/A |
-| **mono_128d** | 128 | 0.8061 | 0.2141 | N/A | N/A |
+| **mono_32d** | 32 | 0.8202 | 0.3403 | N/A | N/A |
+| **mono_64d** | 64 | 0.8230 🏆 | 0.2693 | N/A | N/A |
+| **mono_128d** | 128 | 0.8049 | 0.2129 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8202 | 0.3495 | 0.1580 | 0.5000 |
+| **aligned_64d** | 64 | 0.8230 | 0.2690 | 0.2540 | 0.6280 |
+| **aligned_128d** | 128 | 0.8049 | 0.2090 | 0.3720 | 0.7380 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_64d with 0.8193 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.2790. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** mono_64d with 0.8230 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.2750. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 37.2% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **-0.358** | Low formulaic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -426,19 +461,19 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Prefixes
 | Prefix | Examples |
 |--------|----------|
-| `-co` | colleges, consagratos, consumible |
-| `-ca` | camelot, caldero, cabellera |
-| `-ma` | mardy, manaba, maeztu |
+| `-co` | comuni, concebitas, complices |
+| `-ca` | cassovia, cazadors, castieillo |
+| `-ma` | macromutacions, mariahilfkirche, maree |
+| `-re` | restauro, reumert, recomendable |
 
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-s` | peraleios, noveciercos, engueradas |
-| `-a` | gina, pátria, aquileya |
-| `-as` | engueradas, febas, alimentadas |
-| `-os` | peraleios, noveciercos, consagratos |
-| `-an` | reflectan, goodman, recolectan |
-| `-es` | detalles, colleges, valses |
+| `-s` | viescas, tomus, biggs |
+| `-a` | abaurrepea, telangana, actuaba |
+| `-as` | viescas, medas, rallatas |
+| `-os` | vientos, rasos, tecnolochicos |
+| `-es` | phoenicopteriformes, gabes, bolcheviques |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -446,18 +481,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `ento` | 1.86x | 126 contexts | lento, rento, vento |
-| `cion` | 1.91x | 110 contexts | scion, nacion, accion |
-| `ient` | 1.67x | 177 contexts | aient, dient, oient |
-| `rago` | 2.01x | 59 contexts | ragot, drago, crago |
-| `ranc` | 1.63x | 140 contexts | ranch, rancó, rance |
-| `enci` | 1.55x | 164 contexts | encia, renci, venciu |
-| `laci` | 2.02x | 49 contexts | lacio, glacio, placid |
-| `nter` | 1.55x | 144 contexts | anter, unter, enter |
-| `obla` | 1.85x | 56 contexts | pobla, dobla, robla |
-| `renc` | 1.66x | 81 contexts | arenc, renci, rencor |
-| `ació` | 1.87x | 47 contexts | ación, nació, fació |
-| `mbre` | 1.66x | 75 contexts | ambre, ombre, mbret |
+| `ento` | 1.72x | 126 contexts | rento, gento, sento |
+| `rago` | 2.03x | 58 contexts | crago, drago, ragot |
+| `ranc` | 1.60x | 141 contexts | rancó, ranch, rance |
+| `enci` | 1.55x | 164 contexts | renci, encia, encies |
+| `obla` | 1.95x | 56 contexts | nobla, robla, pobla |
+| `renc` | 1.77x | 82 contexts | renci, arenc, wrench |
+| `ació` | 2.02x | 47 contexts | nació, fació, ación |
+| `ient` | 1.50x | 176 contexts | oient, cient, aient |
+| `nter` | 1.55x | 146 contexts | anter, enter, unter |
+| `cion` | 1.56x | 110 contexts | scion, accion, nacion |
+| `ncia` | 1.69x | 61 contexts | encia, uncia, oencia |
+| `mbre` | 1.55x | 75 contexts | mbret, ambre, umbre |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -465,16 +500,16 @@ This table shows which prefixes and suffixes most frequently co-occur on the sam
 
 | Prefix | Suffix | Frequency | Examples |
 |--------|--------|-----------|----------|
-| `-co` | `-s` | 61 words | conformes, columbus |
-| `-co` | `-a` | 58 words | comunicaba, conservata |
-| `-ca` | `-s` | 54 words | caseilles, cavalls |
-| `-ca` | `-a` | 46 words | carezza, carabaza |
-| `-ma` | `-a` | 38 words | maquinista, mavumengwana |
-| `-ma` | `-s` | 31 words | mamuts, mads |
-| `-ca` | `-as` | 17 words | carpaticas, castanyas |
-| `-co` | `-as` | 16 words | conillas, coladas |
-| `-ca` | `-os` | 12 words | carpos, capuzamientos |
-| `-co` | `-es` | 11 words | conformes, contimparables |
+| `-ca` | `-s` | 64 words | cavens, cabaixos |
+| `-co` | `-s` | 62 words | conejares, concordias |
+| `-co` | `-a` | 57 words | condeixa, cogota |
+| `-ma` | `-s` | 52 words | manganés, manus |
+| `-ca` | `-a` | 44 words | camberra, carola |
+| `-re` | `-s` | 33 words | relationships, refusadas |
+| `-re` | `-a` | 26 words | representa, relacionada |
+| `-ma` | `-a` | 23 words | maganza, magalona |
+| `-co` | `-as` | 19 words | concordias, contadas |
+| `-ca` | `-os` | 16 words | cabaixos, calibos |
 
 ### 6.5 Recursive Morpheme Segmentation
 
@@ -482,26 +517,26 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
-| roncaleses | **`roncal-es-es`** | 6.0 | `roncal` |
-| coinciden | **`co-inciden`** | 4.5 | `inciden` |
-| processos | **`process-os`** | 4.5 | `process` |
-| productoras | **`productor-as`** | 4.5 | `productor` |
-| cobianchi | **`co-bianchi`** | 4.5 | `bianchi` |
-| elefantes | **`elefant-es`** | 4.5 | `elefant` |
-| musicales | **`musical-es`** | 4.5 | `musical` |
-| capelleta | **`ca-pelleta`** | 4.5 | `pelleta` |
-| lumerosas | **`lumer-os-as`** | 3.0 | `lumer` |
-| cantalojas | **`ca-ntaloj-as`** | 3.0 | `ntaloj` |
-| confiscatos | **`co-nfiscat-os`** | 3.0 | `nfiscat` |
-| concentraban | **`co-ncentrab-an`** | 3.0 | `ncentrab` |
-| cavanilles | **`ca-vanill-es`** | 3.0 | `vanill` |
-| aldeyanos | **`aldey-an-os`** | 3.0 | `aldey` |
-| cascavillos | **`ca-scavill-os`** | 3.0 | `scavill` |
+| lombardes | **`lombard-es`** | 4.5 | `lombard` |
+| fanaticos | **`fanatic-os`** | 4.5 | `fanatic` |
+| retransmite | **`re-transmite`** | 4.5 | `transmite` |
+| dimitrios | **`dimitri-os`** | 4.5 | `dimitri` |
+| terroristas | **`terrorist-as`** | 4.5 | `terrorist` |
+| normandas | **`normand-as`** | 4.5 | `normand` |
+| castellan | **`ca-stellan`** | 4.5 | `stellan` |
+| coorganización | **`co-organización`** | 4.5 | `organización` |
+| tortosinas | **`tortosin-as`** | 4.5 | `tortosin` |
+| reportaje | **`re-portaje`** | 4.5 | `portaje` |
+| requiestas | **`re-quiest-as`** | 3.0 | `quiest` |
+| consumitos | **`co-nsumit-os`** | 3.0 | `nsumit` |
+| califatos | **`ca-lifat-os`** | 3.0 | `lifat` |
+| conservamos | **`co-nservam-os`** | 3.0 | `nservam` |
+| colonizatos | **`co-lonizat-os`** | 3.0 | `lonizat` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language AN appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Aragonese shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
 
 ---
 ## 7. Summary & Recommendations
@@ -512,8 +547,8 @@ The language AN appears to be more isolating or has a highly fixed vocabulary. W
 
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
-| Tokenizer | **64k BPE** | Best compression (4.27x) |
-| N-gram | **2-gram** | Lowest perplexity (258) |
+| Tokenizer | **64k BPE** | Best compression (4.28x) |
+| N-gram | **2-gram** | Lowest perplexity (257) |
 | Markov | **Context-4** | Highest predictability (92.6%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
@@ -728,4 +763,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 05:38:36*
+*Report Date: 2026-01-03 14:50:06*

models/embeddings/aligned/an_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fd4098fa2ab7a020113c01a17200837ba8ac73638e61c8edb659c85aaef93fc
+size 1149568062

models/embeddings/aligned/an_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "an", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/an_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd99fd2982f86d44f39d6905f1bab4d510cd2a435654c9497853d449b81da158
+size 65664

models/embeddings/aligned/an_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "an",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 46611,
+  "vocab_size": 120553
+}

models/embeddings/aligned/an_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8162f33f0e10366ed7d3fdfb8fa216d05dbd6d4a16b2198859372aa289d98b14
+size 288983358

models/embeddings/aligned/an_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "an", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/an_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:353d511257321af48a3269657d78455f93120c5fcf17d70f5f3b6de72a95f856
+size 4224

models/embeddings/aligned/an_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "an",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 46611,
+  "vocab_size": 120553
+}

models/embeddings/aligned/an_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95a4be660051d6fcd8c79a595fd6237cd7f54790bfab31f65b7a8f9c2533a8e2
+size 575844926

models/embeddings/aligned/an_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "an", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/an_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a69faaf2fde42a4fb0c85609a8ab9ff29cffe1093a3932f0aa3d238f32d74f9
+size 16512

models/embeddings/aligned/an_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "an",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 46611,
+  "vocab_size": 120553
+}

models/embeddings/monolingual/an_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b8c8d2f8e8ebd6d647e7092d20c2a5ed3abae1a07a6be40fc84c516534b000d
-size 1148906690
+oid sha256:5fd4098fa2ab7a020113c01a17200837ba8ac73638e61c8edb659c85aaef93fc
+size 1149568062

models/embeddings/monolingual/an_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 119918
+  "vocab_size": 120553
 }

models/embeddings/monolingual/an_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:227c09966b020164fd97d17f0e971bef21897af0662c9891e19c26eef18f815d
-size 288809666
+oid sha256:8162f33f0e10366ed7d3fdfb8fa216d05dbd6d4a16b2198859372aa289d98b14
+size 288983358

models/embeddings/monolingual/an_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 119918
+  "vocab_size": 120553
 }

models/embeddings/monolingual/an_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:168f92e58a097a484a5136a652e39c7b532d3e83d40af551f9b434d64c32dcf2
-size 575508674
+oid sha256:95a4be660051d6fcd8c79a595fd6237cd7f54790bfab31f65b7a8f9c2533a8e2
+size 575844926

models/embeddings/monolingual/an_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 119918
+  "vocab_size": 120553
 }

models/subword_markov/an_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d40de84446df679c4cebd27cec50dd996c043fbace52eee0439eb6badc7bd7c
-size 168642
+oid sha256:82737fda9661e7c5ecf12fa0a67a2d2bcc18af9b5e9e30a7b0158eef5c952909
+size 176131

models/subword_markov/an_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "an",
-  "unique_contexts": 3499,
-  "total_transitions": 69109076
+  "unique_contexts": 3672,
+  "total_transitions": 69810476
 }

models/subword_markov/an_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49319842d520d6479015d48b0398bc41858ef988fef862a150994662fcc06982
-size 916009
+oid sha256:f0fc7fa0b6576fa1c527465494c2d88de956a333230c36d23818149db6ffd093
+size 938570

models/subword_markov/an_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "an",
-  "unique_contexts": 20583,
-  "total_transitions": 69059397
+  "unique_contexts": 21123,
+  "total_transitions": 69760181
 }

models/subword_markov/an_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:706cb13def2cb1c8316e030700d89c8a6b8edcdaa283466b09d5fe99ab709294
-size 3807898
+oid sha256:002be2f1eb632f8e6574678f853e93f4bbca357d8fe282182dc11766db2955b4
+size 3816590

models/subword_markov/an_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "an",
-  "unique_contexts": 109746,
-  "total_transitions": 69009718
+  "unique_contexts": 110486,
+  "total_transitions": 69709886
 }

models/subword_markov/an_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5eba517deec4e418e88d7689e4203c7c287c5ff4a05060f0fbb2342c404dccc8
-size 13264812
+oid sha256:1a8a70c3406dca6ea2afc3bd1a4133f81559c04cc64ebb35b40bd88bf0512e2e
+size 13281906

models/subword_markov/an_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "an",
-  "unique_contexts": 474426,
-  "total_transitions": 68960039
+  "unique_contexts": 474961,
+  "total_transitions": 69659591
 }

models/subword_ngram/an_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:555bb0a9950d43d338c420105f45d6e40844826ad59879138e4521a5e4f7bc0c
-size 93665
+oid sha256:0f6ff5966a58dcbf3f78684a10d07bc43305bed96d2231b9a562c78ffe158f5e
+size 93856

models/subword_ngram/an_2gram_subword_metadata.json

@@ -3,5 +3,5 @@
   "variant": "subword",
   "language": "an",
   "unique_ngrams": 7000,
-  "total_ngrams": 69109076
+  "total_ngrams": 69810476
 }

models/subword_ngram/an_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f78a3129dc57d3cdb79e700b2f8fd170e91b6a063ad7ecc8c3970aa700eb2eba
-size 691123
+oid sha256:2b9410a959938ec2ee19d3d785fdbe7c03b9888b017ac84c0907cc567cc67382
+size 690365

models/subword_ngram/an_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "an",
-  "unique_ngrams": 52839,
-  "total_ngrams": 69059397
+  "unique_ngrams": 52727,
+  "total_ngrams": 69760181
 }

models/subword_ngram/an_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c2ad5d14d9c914207a8289e32db63c141e19a1ca414e8a0d26bb2bd2d7c2251
-size 3358033
+oid sha256:54766951ed2a769fc5227d30241529307ad56476f75a1c6ded8dfd17cfd005c6
+size 3361576

models/subword_ngram/an_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "an",
-  "unique_ngrams": 289840,
-  "total_ngrams": 69009718
+  "unique_ngrams": 289768,
+  "total_ngrams": 69709886
 }

models/subword_ngram/an_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2d2a42c46ff9f023d7a1e77f90126884ff7127984bdea7beb92f1a4ea9ba286
+size 10691226

models/subword_ngram/an_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "an",
+  "unique_ngrams": 901225,
+  "total_ngrams": 69659591
+}

models/tokenizer/an_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e6718ee086df807a9b8e62706d7c765c76896efe91b3e0c87de4d756afbca43
-size 511370
+oid sha256:be4e4495dd91b35e24a5508fe0c746604c258fa5e24d680caa9984c3edca42a6
+size 511374

models/tokenizer/an_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a3ebce2687e88197aaa7a091e4f04cb995c75343de7c69b2a452444c0e4ec12
-size 791243
+oid sha256:5fa703683fed7866aee835ffdc13a44b7a714de834a38b62b70558897787c1fb
+size 791864

models/tokenizer/an_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7137d416046b5467fc533de58fb10d2dfc01761285b5489a24416366e18446ce
-size 1364521
+oid sha256:f1989e3ff85462c98efc25880434afda45544dd0dff0bda857ccfabf9d8725dc
+size 1363420

models/tokenizer/an_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6b38bd103587bec698ef8bfa6fc1eca198cf371a05427efd0b9100615abdbd5
-size 374587
+oid sha256:24cd47460a4e12244cf3b7f7de3902be403e0466c77da2537d5c747ecd2e8c4d
+size 374577

models/vocabulary/an_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:899ce817bf3d0cd63c52370382d0c8f7e06c5beb0c78ed09f131cf1aafd3d182
-size 2932502
+oid sha256:6edbfc40a426299cd59549a074009433199230abdf905b5190070b6c0275b690
+size 2944293

models/vocabulary/an_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "an",
-  "vocabulary_size": 182961,
+  "vocabulary_size": 183928,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.03129810058501218,
+    "type_token_ratio": 0.031130701540467178,
     "coverage": {
-      "top_100": 0.4415954090654393,
-      "top_1000": 0.6581393006575422,
-      "top_5000": 0.793948937884549,
-      "top_10000": 0.8454613838789675
+      "top_100": 0.4412633632212311,
+      "top_1000": 0.6580607382657693,
+      "top_5000": 0.7940228593838857,
+      "top_10000": 0.8455141690008805
     },
-    "hapax_count": 184348,
-    "hapax_ratio": 0.50188805610535,
-    "total_documents": 49679
+    "hapax_count": 184865,
+    "hapax_ratio": 0.5012703603376419,
+    "total_documents": 50295
   }
 }

models/word_markov/an_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:24ca3e38b9546b9a06f5df3b46ba1a09d874251e2d187e8bdb92e634b9a97756
-size 23502911
+oid sha256:7b2b7d443bdb24601458b682628e19a693091fcc060312b8c29b8d0aeca42d42
+size 23695943

models/word_markov/an_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "an",
-  "unique_contexts": 367064,
-  "total_transitions": 11686145
+  "unique_contexts": 368549,
+  "total_transitions": 11796306
 }

models/word_markov/an_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c73b675412ec88a37c27f3ed414e5087949843a731d927309077f06614513901
-size 65110098
+oid sha256:f8a94e65ec2e1cbb3239d99357349f5de3f99c8f328da97ec36993f6bcaf9b2c
+size 65492964

models/word_markov/an_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "an",
-  "unique_contexts": 2775765,
-  "total_transitions": 11636466
+  "unique_contexts": 2791626,
+  "total_transitions": 11746011
 }