Upload all models and assets for ast (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: ast
-language_name: AST
+language_name: Asturian
 language_family: romance_iberian
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-romance_iberian
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.427
+    value: 4.429
   - name: best_isotropy
     type: isotropy
-    value: 0.7909
+    value: 0.7932
   - name: vocabulary_size
     type: vocab
     value: 0
-generated: 2026-01-03
+generated: 2026-01-04
 ---
 
-# AST - Wikilangs Models
+# Asturian - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **AST** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Asturian** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.569x | 3.57 | 0.0259% | 871,221 |
-| **16k** | 3.921x | 3.92 | 0.0285% | 793,006 |
-| **32k** | 4.204x | 4.21 | 0.0306% | 739,567 |
-| **64k** | 4.427x 🏆 | 4.43 | 0.0322% | 702,254 |
+| **8k** | 3.571x | 3.57 | 0.0264% | 863,429 |
+| **16k** | 3.921x | 3.92 | 0.0290% | 786,292 |
+| **32k** | 4.205x | 4.21 | 0.0311% | 733,251 |
+| **64k** | 4.429x 🏆 | 4.43 | 0.0327% | 696,255 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Luiz Diallisson de Souza Alves ye un futbolista brasilanu. Clubes Kuban Referenc...`
+**Sample 1:** `Pol nome de Pedru'l Grande conocemos a dos monarques europeos: Pedru III d'Aragó...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁lu iz ▁di all is son ▁de ▁sou za ▁al ... (+14 more)` | 24 |
-| 16k | `▁lu iz ▁di all is son ▁de ▁sou za ▁al ... (+14 more)` | 24 |
-| 32k | `▁luiz ▁di all is son ▁de ▁souza ▁alves ▁ye ▁un ... (+11 more)` | 21 |
-| 64k | `▁luiz ▁di all isson ▁de ▁souza ▁alves ▁ye ▁un ▁futbolista ... (+10 more)` | 20 |
+| 8k | `▁pol ▁nome ▁de ▁ped ru ' l ▁grande ▁cono ce ... (+21 more)` | 31 |
+| 16k | `▁pol ▁nome ▁de ▁pedru ' l ▁grande ▁cono ce mos ... (+18 more)` | 28 |
+| 32k | `▁pol ▁nome ▁de ▁pedru ' l ▁grande ▁conocemos ▁a ▁dos ... (+15 more)` | 25 |
+| 64k | `▁pol ▁nome ▁de ▁pedru ' l ▁grande ▁conocemos ▁a ▁dos ... (+15 more)` | 25 |
 
-**Sample 2:** `Vagner da Silva Sarti ye un ex-futbolista brasilanu. Clubes Referencies Enllaces...`
+**Sample 2:** `Yuki Ohashi (, ) ye un futbolista xaponés. Clubes Referencies Enllaces esternos ...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁va gn er ▁da ▁silva ▁sar ti ▁ye ▁un ▁ex ... (+10 more)` | 20 |
-| 16k | `▁va gner ▁da ▁silva ▁sar ti ▁ye ▁un ▁ex - ... (+9 more)` | 19 |
-| 32k | `▁va gner ▁da ▁silva ▁sar ti ▁ye ▁un ▁ex - ... (+9 more)` | 19 |
-| 64k | `▁va gner ▁da ▁silva ▁sar ti ▁ye ▁un ▁ex - ... (+9 more)` | 19 |
+| 8k | `▁yu ki ▁oh as hi ▁(, ▁) ▁ye ▁un ▁futbolista ... (+14 more)` | 24 |
+| 16k | `▁yu ki ▁oh ashi ▁(, ▁) ▁ye ▁un ▁futbolista ▁xaponés ... (+12 more)` | 22 |
+| 32k | `▁yuki ▁oh ashi ▁(, ▁) ▁ye ▁un ▁futbolista ▁xaponés . ... (+11 more)` | 21 |
+| 64k | `▁yuki ▁oh ashi ▁(, ▁) ▁ye ▁un ▁futbolista ▁xaponés . ... (+11 more)` | 21 |
 
-**Sample 3:** `(MMLXXXII) va ser un añu normal entamáu en xueves nel calendariu gregorianu. Ref...`
+**Sample 3:** `Fechos Nacencies Muertes Referencies Enllaces esternos V e.C.`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁( m m l xx x ii ) ▁va ▁ser ... (+17 more)` | 27 |
-| 16k | `▁( mm l xx x ii ) ▁va ���ser ▁un ... (+14 more)` | 24 |
-| 32k | `▁( mm l xx xii ) ▁va ▁ser ▁un ▁añu ... (+12 more)` | 22 |
-| 64k | `▁( mm lxx xii ) ▁va ▁ser ▁un ▁añu ▁normal ... (+11 more)` | 21 |
+| 8k | `▁fechos ▁nacencies ▁muertes ▁referencies ▁enllaces ▁esternos ▁v ▁e . c ... (+1 more)` | 11 |
+| 16k | `▁fechos ▁nacencies ▁muertes ▁referencies ▁enllaces ▁esternos ▁v ▁e . c ... (+1 more)` | 11 |
+| 32k | `▁fechos ▁nacencies ▁muertes ▁referencies ▁enllaces ▁esternos ▁v ▁e . c ... (+1 more)` | 11 |
+| 64k | `▁fechos ▁nacencies ▁muertes ▁referencies ▁enllaces ▁esternos ▁v ▁e . c ... (+1 more)` | 11 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.427x compression
-- **Lowest UNK Rate:** 8k with 0.0259% unknown tokens
+- **Best Compression:** 64k achieves 4.429x compression
+- **Lowest UNK Rate:** 8k with 0.0264% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 133,027 | 17.02 | 1,354,323 | 9.8% | 21.6% |
-| **2-gram** | Subword | 260 🏆 | 8.02 | 19,069 | 69.7% | 99.1% |
-| **3-gram** | Word | 646,899 | 19.30 | 2,908,394 | 4.2% | 10.7% |
-| **3-gram** | Subword | 2,223 | 11.12 | 139,212 | 28.0% | 72.3% |
-| **4-gram** | Word | 1,559,764 | 20.57 | 4,707,856 | 3.3% | 7.5% |
-| **4-gram** | Subword | 13,372 | 13.71 | 791,795 | 13.9% | 39.3% |
+| **2-gram** | Word | 132,138 | 17.01 | 1,341,882 | 9.8% | 21.7% |
+| **2-gram** | Subword | 260 🏆 | 8.02 | 19,027 | 69.7% | 99.1% |
+| **3-gram** | Word | 640,312 | 19.29 | 2,878,367 | 4.2% | 10.7% |
+| **3-gram** | Subword | 2,218 | 11.12 | 138,526 | 28.0% | 72.3% |
+| **4-gram** | Word | 1,536,908 | 20.55 | 4,654,291 | 3.3% | 7.6% |
+| **4-gram** | Subword | 13,337 | 13.70 | 787,142 | 13.9% | 39.3% |
+| **5-gram** | Word | 1,050,558 | 20.00 | 2,949,427 | 4.8% | 9.6% |
+| **5-gram** | Subword | 57,630 | 15.81 | 2,701,102 | 7.8% | 23.5% |
 
 ### Top 5 N-grams by Size
 
@@ -150,68 +162,88 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `de la` | 891,402 |
-| 2 | `de los` | 329,410 |
-| 3 | `la so` | 220,083 |
-| 4 | `a la` | 215,036 |
-| 5 | `de les` | 208,071 |
+| 1 | `de la` | 877,001 |
+| 2 | `de los` | 325,167 |
+| 3 | `la so` | 218,605 |
+| 4 | `a la` | 213,098 |
+| 5 | `de les` | 205,401 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `referencies enllaces esternos` | 101,643 |
-| 2 | `de la so` | 48,838 |
-| 3 | `d estaos xuníos` | 34,333 |
-| 4 | `enllaces esternos de` | 33,237 |
-| 5 | `una población de` | 30,269 |
+| 1 | `referencies enllaces esternos` | 102,198 |
+| 2 | `de la so` | 48,437 |
+| 3 | `d estaos xuníos` | 34,372 |
+| 4 | `enllaces esternos de` | 33,442 |
+| 5 | `una población de` | 30,281 |
 
 **4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `referencies enllaces esternos de` | 32,314 |
-| 2 | `tien una población de` | 26,720 |
-| 3 | `una población de y` | 19,598 |
+| 1 | `referencies enllaces esternos de` | 32,439 |
+| 2 | `tien una población de` | 26,725 |
+| 3 | `una población de y` | 19,595 |
 | 4 | `y una superficie de` | 19,554 |
-| 5 | `una superficie de km` | 19,519 |
+| 5 | `población de y una` | 19,514 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `tien una población de y` | 19,555 |
+| 2 | `una población de y una` | 19,513 |
+| 3 | `de y una superficie de` | 19,492 |
+| 4 | `población de y una superficie` | 19,490 |
+| 5 | `y una superficie de km` | 19,254 |
 
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `a _` | 12,346,491 |
-| 2 | `e _` | 10,275,492 |
-| 3 | `s _` | 10,054,248 |
-| 4 | `_ d` | 9,863,919 |
-| 5 | `e s` | 9,411,923 |
+| 1 | `a _` | 12,223,314 |
+| 2 | `e _` | 10,169,137 |
+| 3 | `s _` | 9,980,231 |
+| 4 | `_ d` | 9,749,761 |
+| 5 | `e s` | 9,339,123 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `_ d e` | 7,215,701 |
-| 2 | `d e _` | 5,349,035 |
-| 3 | `e s _` | 4,769,369 |
-| 4 | `o s _` | 3,909,790 |
-| 5 | `l a _` | 3,068,189 |
+| 1 | `_ d e` | 7,125,386 |
+| 2 | `d e _` | 5,278,423 |
+| 3 | `e s _` | 4,734,999 |
+| 4 | `o s _` | 3,881,527 |
+| 5 | `l a _` | 3,034,851 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `_ d e _` | 4,975,922 |
-| 2 | `_ l a _` | 2,468,941 |
-| 3 | `d e _ l` | 1,667,072 |
-| 4 | `a _ d e` | 1,422,241 |
-| 5 | `s _ d e` | 1,380,334 |
+| 1 | `_ d e _` | 4,909,705 |
+| 2 | `_ l a _` | 2,443,055 |
+| 3 | `d e _ l` | 1,642,151 |
+| 4 | `a _ d e` | 1,399,483 |
+| 5 | `s _ d e` | 1,367,031 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `_ d e _ l` | 1,593,336 |
+| 2 | `e _ l a _` | 1,090,094 |
+| 3 | `_ d e l _` | 1,070,352 |
+| 4 | `s _ d e _` | 1,000,253 |
+| 5 | `a _ d e _` | 970,617 |
 
 
 ### Key Findings
 
 - **Best Perplexity:** 2-gram (subword) with 260
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~39% of corpus
+- **Coverage:** Top-1000 patterns cover ~23% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 1.0381 | 2.054 | 12.99 | 1,204,316 | 0.0% |
-| **1** | Subword | 1.1983 | 2.295 | 7.97 | 10,463 | 0.0% |
-| **2** | Word | 0.4193 | 1.337 | 2.57 | 15,634,564 | 58.1% |
-| **2** | Subword | 0.6558 | 1.576 | 4.28 | 83,437 | 34.4% |
-| **3** | Word | 0.1865 | 1.138 | 1.44 | 40,202,890 | 81.4% |
-| **3** | Subword | 0.6846 | 1.607 | 4.03 | 357,207 | 31.5% |
-| **4** | Word | 0.0789 🏆 | 1.056 | 1.15 | 57,817,277 | 92.1% |
-| **4** | Subword | 0.6846 | 1.607 | 3.51 | 1,439,883 | 31.5% |
+| **1** | Word | 1.0362 | 2.051 | 12.93 | 1,199,957 | 0.0% |
+| **1** | Subword | 1.1986 | 2.295 | 7.97 | 10,438 | 0.0% |
+| **2** | Word | 0.4189 | 1.337 | 2.57 | 15,504,920 | 58.1% |
+| **2** | Subword | 0.6561 | 1.576 | 4.28 | 83,238 | 34.4% |
+| **3** | Word | 0.1863 | 1.138 | 1.44 | 39,817,744 | 81.4% |
+| **3** | Subword | 0.6835 | 1.606 | 4.02 | 356,042 | 31.6% |
+| **4** | Word | 0.0788 🏆 | 1.056 | 1.14 | 57,235,451 | 92.1% |
+| **4** | Subword | 0.6840 | 1.607 | 3.51 | 1,432,910 | 31.6% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `de xunu empieza a centros multimodales funciones nel nectariu semilunar o n ucraín y comunicaciones ...`
-2. `la pieza cornelius coffin fit cuando el algebasó 503 mariña d una solución bonal o nun`
-3. `y 15 m sobre l uniforme del postreru gran midida china tales from here mirror weekly`
+1. `de teniente xeneral del planeta mientres la realidá quiciabes d ochobre foi escritu por aciu una`
+2. `la cual el propósitu un estilu y hornsby consiguieron 31 d alabama intentó nun tour a`
+3. `y derechos humanos ta estremada en determinóse que caltener la so home l minsiterio de candela`
 
 **Context Size 2:**
 
-1. `de la provincia dende esti tornéu surdió en y persuadió a eliza dushku en películes d estudiante`
-2. `de los documentos relativos al mercáu l so antiguu nome dau más tarde l empresariu estremeñu dueñu`
-3. `la so base na isla parker llogró atrapar la pelota vasca que se llevó a empecipiar una`
+1. `de la cocina nos años y escuchar música dende la edá kim young chae sbs jumpmbc nonstop`
+2. `de los fundadores de los cinco principales epítetos y títulos descriptivos de los chola fueron movío...`
+3. `la so bona contrarreló calteniendo a dellos decretos prohibiendo la llibre asociación como ye l cuan...`
 
 **Context Size 3:**
 
-1. `referencies enllaces esternos el salín nel suelu y ente vexetación trupa sicasí en marismas y ribere...`
-2. `de la so agua h havagazı gas o otobüs bus y t troleybüs trolebús magar que los entamos`
-3. `enllaces esternos de côte d or na rexón de gran este llenda con tien una población de 1`
+1. `referencies enllaces esternos green breasted mangu english wikipedia consultáu l 2 de marzu de estab...`
+2. `de la so política d esclusión nel sieglu xx en que camudó de nome los líderes del movimientu`
+3. `enllaces esternos de xapón de la prefeutura de hyogo llocalización con una superficie de km ver tami...`
 
 **Context Size 4:**
 
-1. `referencies enllaces esternos de saboya de francia de bretaña de dreux de bretaña`
-2. `tien una población de 1 690 471 habitantes y un puertu fluvial sobre l paraná amás tien importancia ...`
-3. `una población de y una superficie de km ver tamién referencies enllaces esternos de xapón de la pref...`
+1. `referencies enllaces esternos de piloña de piloña`
+2. `tien una población de y una superficie de km y una población de referencies enllaces esternos de xap...`
+3. `una población de y una superficie de km referencies enllaces esternos d aquila`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `_fén_an_yaconyíc`
-2. `er,_ciesunton_a_`
-3. `a_tostelociz_ce_`
+1. `_eral_r_s_de_prm`
+2. `el_untodesopay_c`
+3. `armbra_a_wozall_`
 
 **Context Size 2:**
 
-1. `a_gasainel_tabaro`
-2. `e_es_de_y_chel_má`
-3. `s_astamudia_de_ll`
+1. `a_dada_d'alicu_de`
+2. `e_al_crein_ings._`
+3. `s_agu_pobres_saos`
 
 **Context Size 3:**
 
-1. `_de_scharacióse_le`
-2. `de_tragar_primera_`
-3. `es_so_títulu_miliz`
+1. `_de_s'atroxina_pa_`
+2. `de_los_nuevu._fíos`
+3. `es_deste_-_frivaes`
 
 **Context Size 4:**
 
-1. `_de_los_sobres_del_`
-2. `_la_ermistoria_dife`
-3. `de_los_xeneia,_cons`
+1. `_de_mouther_de_fort`
+2. `_la_cada_y_márquist`
+3. `de_la_sociedá_nacio`
 
 
 ### Key Findings
 
 - **Best Predictability:** Context-4 (word) with 92.1% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (1,439,883 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (1,432,910 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,48 +346,48 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 555,056 |
-| Total Tokens | 75,071,637 |
-| Mean Frequency | 135.25 |
+| Vocabulary Size | 552,425 |
+| Total Tokens | 74,325,511 |
+| Mean Frequency | 134.54 |
 | Median Frequency | 4 |
-| Frequency Std Dev | 9337.66 |
+| Frequency Std Dev | 9254.05 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | de | 4,994,843 |
-| 2 | la | 2,512,518 |
-| 3 | y | 2,055,358 |
-| 4 | d | 1,181,646 |
-| 5 | a | 1,163,388 |
-| 6 | del | 1,091,464 |
-| 7 | en | 1,070,328 |
-| 8 | que | 1,013,684 |
-| 9 | los | 966,280 |
-| 10 | l | 958,680 |
+| 1 | de | 4,928,261 |
+| 2 | la | 2,485,426 |
+| 3 | y | 2,042,239 |
+| 4 | d | 1,169,053 |
+| 5 | a | 1,155,083 |
+| 6 | del | 1,074,281 |
+| 7 | en | 1,055,986 |
+| 8 | que | 1,007,870 |
+| 9 | los | 957,887 |
+| 10 | l | 950,908 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | sverlo | 2 |
-| 2 | kmca | 2 |
-| 3 | antimaterialistas | 2 |
-| 4 | infectados | 2 |
-| 5 | historietistas | 2 |
-| 6 | curtmetratxe | 2 |
-| 7 | rugna | 2 |
-| 8 | lleáu | 2 |
-| 9 | queña | 2 |
-| 10 | nkoghe | 2 |
+| 1 | leptafeke | 2 |
+| 2 | haua | 2 |
+| 3 | küzdoblani | 2 |
+| 4 | contrarrellatu | 2 |
+| 5 | semilleru | 2 |
+| 6 | bisterca | 2 |
+| 7 | šafarsko | 2 |
+| 8 | vyfalu | 2 |
+| 9 | ribich | 2 |
+| 10 | lacos | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 0.9991 |
-| R² (Goodness of Fit) | 0.995555 |
+| Zipf Coefficient | 0.9990 |
+| R² (Goodness of Fit) | 0.995611 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
@@ -371,7 +403,7 @@ Below are text samples generated from each subword-based Markov chain model:
 
 - **Zipf Compliance:** R²=0.9956 indicates excellent adherence to Zipf's law
 - **High Frequency Dominance:** Top 100 words cover 41.7% of corpus
-- **Long Tail:** 545,056 words needed for remaining 16.9% coverage
+- **Long Tail:** 542,425 words needed for remaining 16.9% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.7909 🏆 | 0.3827 | N/A | N/A |
-| **mono_64d** | 64 | 0.7802 | 0.3065 | N/A | N/A |
-| **mono_128d** | 128 | 0.7192 | 0.2391 | N/A | N/A |
+| **mono_32d** | 32 | 0.7932 | 0.3820 | N/A | N/A |
+| **mono_64d** | 64 | 0.7818 | 0.2979 | N/A | N/A |
+| **mono_128d** | 128 | 0.7210 | 0.2388 | N/A | N/A |
+| **aligned_32d** | 32 | 0.7932 🏆 | 0.3922 | 0.3820 | 0.7300 |
+| **aligned_64d** | 64 | 0.7818 | 0.3048 | 0.5840 | 0.8840 |
+| **aligned_128d** | 128 | 0.7210 | 0.2380 | 0.7080 | 0.9240 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.7909 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.3094. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** aligned_32d with 0.7932 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.3090. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 70.8% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **-0.591** | Low formulaic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -426,22 +461,19 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Prefixes
 | Prefix | Examples |
 |--------|----------|
-| `-co` | comíen, compelidos, conciliable |
-| `-ma` | maravíase, maça, matematización |
-| `-re` | reescalada, reprimió, reconociéralu |
-| `-de` | deduz, declaratorio, desfila |
-| `-ca` | caminómetru, castromil, caecilia |
+| `-co` | control, coetzee, conversión |
+| `-ma` | manifiéstase, marinel, matraqueo |
+| `-re` | rehnskiöld, rendimientos, reichholf |
 
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-s` | phrygilus, anticolinérgicos, friulianos |
-| `-a` | raksasa, estendería, reescalada |
-| `-es` | ibes, distopíes, ziríes |
-| `-os` | anticolinérgicos, friulianos, afogadiegos |
-| `-se` | esmoreciérase, maravíase, cuayábase |
-| `-as` | monarquistas, gorgas, mimeografiadas |
-| `-en` | altshausen, comíen, blegen |
+| `-s` | narganes, supracaudales, señálennos |
+| `-a` | carga, balsámica, trueba |
+| `-es` | narganes, supracaudales, rastres |
+| `-os` | señálennos, sabéivos, visos |
+| `-se` | escapóse, ñublense, manifiéstase |
+| `-as` | tankas, aleutas, ḥechas |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -449,18 +481,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `iend` | 1.80x | 206 contexts | fiend, iendo, viendi |
-| `renc` | 2.05x | 99 contexts | frenc, wrench, rencor |
-| `ient` | 1.67x | 271 contexts | vient, iente, aient |
-| `enci` | 1.52x | 262 contexts | venci, benci, cenci |
-| `acio` | 1.63x | 166 contexts | nacio, cacio, tacio |
-| `ació` | 1.79x | 94 contexts | lació, xació, ñació |
-| `nter` | 1.38x | 335 contexts | inter, enter, unter |
-| `ontr` | 1.63x | 118 contexts | contr, contra, montra |
-| `ener` | 1.42x | 205 contexts | enerc, tener, enero |
-| `ntos` | 1.79x | 67 contexts | antos, entos, tintos |
-| `ntes` | 1.49x | 144 contexts | antes, entes, fontes |
-| `efer` | 1.61x | 86 contexts | sefer, nefer, refer |
+| `iend` | 1.75x | 206 contexts | fiend, iendo, rienda |
+| `ació` | 1.96x | 92 contexts | ñació, lació, xació |
+| `ogra` | 1.57x | 189 contexts | logra, bogra, sogra |
+| `ient` | 1.46x | 273 contexts | iente, cient, aient |
+| `acio` | 1.55x | 167 contexts | bacio, facio, macio |
+| `renc` | 1.71x | 99 contexts | frenc, lorenc, trench |
+| `ntes` | 1.56x | 144 contexts | antes, entes, entesa |
+| `enci` | 1.35x | 261 contexts | encia, cenci, venci |
+| `efer` | 1.63x | 86 contexts | refer, defer, sefer |
+| `ntos` | 1.72x | 67 contexts | antos, entos, tantos |
+| `raci` | 1.41x | 164 contexts | racib, racio, iraci |
+| `ontr` | 1.50x | 117 contexts | contr, kontra, lontra |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -468,16 +500,16 @@ This table shows which prefixes and suffixes most frequently co-occur on the sam
 
 | Prefix | Suffix | Frequency | Examples |
 |--------|--------|-----------|----------|
-| `-co` | `-s` | 59 words | conversas, concinnus |
-| `-ca` | `-s` | 53 words | cancelaciones, caloiros |
-| `-ca` | `-a` | 49 words | cartajima, campana |
-| `-co` | `-a` | 44 words | comella, copia |
-| `-ma` | `-a` | 38 words | matina, matrioshka |
-| `-re` | `-s` | 34 words | rectos, restaurantes |
-| `-ma` | `-s` | 31 words | maniobres, maderensis |
-| `-de` | `-s` | 31 words | descatados, definitives |
-| `-co` | `-es` | 25 words | cotidales, coleicionables |
-| `-re` | `-a` | 24 words | renombraría, retomara |
+| `-co` | `-s` | 55 words | consentimientos, correllaciones |
+| `-ma` | `-a` | 44 words | maniobraba, marra |
+| `-ma` | `-s` | 40 words | macromicetes, maorís |
+| `-re` | `-a` | 39 words | reflorestada, respondida |
+| `-co` | `-a` | 37 words | comitia, cornigera |
+| `-re` | `-s` | 33 words | refundiándoles, reprogramables |
+| `-re` | `-se` | 27 words | reproducense, retomándose |
+| `-co` | `-es` | 23 words | correllaciones, coeditores |
+| `-co` | `-se` | 22 words | confiándose, comercializábense |
+| `-re` | `-es` | 20 words | refundiándoles, reprogramables |
 
 ### 6.5 Recursive Morpheme Segmentation
 
@@ -485,26 +517,26 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
-| retractores | **`re-tractor-es`** | 6.0 | `tractor` |
-| aseguráronse | **`aseguráron-se`** | 4.5 | `aseguráron` |
-| tendiéronse | **`tendiéron-se`** | 4.5 | `tendiéron` |
-| tresversales | **`tresversal-es`** | 4.5 | `tresversal` |
-| redefiniéronse | **`re-de-finiéron-se`** | 4.5 | `finiéron` |
-| redistributivo | **`re-distributivo`** | 4.5 | `distributivo` |
-| prométese | **`prométe-se`** | 4.5 | `prométe` |
-| escaecíen | **`escaecí-en`** | 4.5 | `escaecí` |
-| domadores | **`domador-es`** | 4.5 | `domador` |
-| consérvense | **`co-nsérv-en-se`** | 4.5 | `nsérv` |
-| descripto | **`de-scripto`** | 4.5 | `scripto` |
-| esaxeróse | **`esaxeró-se`** | 4.5 | `esaxeró` |
-| acentores | **`acentor-es`** | 4.5 | `acentor` |
-| detrayendo | **`de-trayendo`** | 4.5 | `trayendo` |
-| renormalización | **`re-normalización`** | 4.5 | `normalización` |
+| clamorosos | **`clamor-os-os`** | 6.0 | `clamor` |
+| doloroses | **`dolor-os-es`** | 6.0 | `dolor` |
+| velenoses | **`velen-os-es`** | 6.0 | `velen` |
+| escribiríase | **`escribiría-se`** | 4.5 | `escribiría` |
+| mundiales | **`mundial-es`** | 4.5 | `mundial` |
+| desgraciaos | **`desgracia-os`** | 4.5 | `desgracia` |
+| alfayates | **`alfayat-es`** | 4.5 | `alfayat` |
+| cristalizase | **`cristaliza-se`** | 4.5 | `cristaliza` |
+| remensura | **`re-mensura`** | 4.5 | `mensura` |
+| desequilibraos | **`desequilibra-os`** | 4.5 | `desequilibra` |
+| decretase | **`decreta-se`** | 4.5 | `decreta` |
+| coartífice | **`co-artífice`** | 4.5 | `artífice` |
+| declaráse | **`declará-se`** | 4.5 | `declará` |
+| reordenar | **`re-ordenar`** | 4.5 | `ordenar` |
+| pediatres | **`pediatr-es`** | 4.5 | `pediatr` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language AST appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Asturian shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
 
 ---
 ## 7. Summary & Recommendations
@@ -731,4 +763,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 09:38:21*
+*Report Date: 2026-01-04 02:53:18*

models/embeddings/aligned/ast_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:117e4af0e5eeebd084921ab7e00ceec3fdaf5da8a39ee75046517fbade337531
+size 1521825637

models/embeddings/aligned/ast_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "ast", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/ast_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0933b97d07adce96c407981abf345cf4da38e52496f04bd5a25340f77949ffc6
+size 65664

models/embeddings/aligned/ast_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "ast",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 153826,
+  "vocab_size": 477621
+}

models/embeddings/aligned/ast_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24116b4c67245399819efd28c41d4e40aeae14cf5daac9443be544975c264282
+size 387012709

models/embeddings/aligned/ast_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "ast", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/ast_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46da77c4ee9c3dbf33f571908ebcc536ec02c7a7c71822d393320c0e2ae81356
+size 4224

models/embeddings/aligned/ast_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "ast",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 153826,
+  "vocab_size": 477621
+}

models/embeddings/aligned/ast_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1f70d7b0780c34fd320fda5cc24a8a07da0788ffd1f0363aa1a63284f4cde73
+size 765283685

models/embeddings/aligned/ast_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "ast", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/ast_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7180fa74480d959c99d9b6e8438aa69d08493a73920b7efe643090e11e0531dc
+size 16512

models/embeddings/aligned/ast_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "ast",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 153826,
+  "vocab_size": 477621
+}

models/embeddings/monolingual/ast_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c4500458da720e9717e0be530291de837dfe1937d937d8f59cd5975214f6d01
-size 1526234273
+oid sha256:117e4af0e5eeebd084921ab7e00ceec3fdaf5da8a39ee75046517fbade337531
+size 1521825637

models/embeddings/monolingual/ast_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 481846
+  "vocab_size": 477621
 }

models/embeddings/monolingual/ast_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa3cdba099d3af49ab89043ad5713c1fe468060d520334f5200b000925a5e30e
-size 388176545
+oid sha256:24116b4c67245399819efd28c41d4e40aeae14cf5daac9443be544975c264282
+size 387012709

models/embeddings/monolingual/ast_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 481846
+  "vocab_size": 477621
 }

models/embeddings/monolingual/ast_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e24d4561e7c9f15e391c915d7d4096c8b8adf794324cd6df9a60e78e6f5fc6a
-size 767529121
+oid sha256:e1f70d7b0780c34fd320fda5cc24a8a07da0788ffd1f0363aa1a63284f4cde73
+size 765283685

models/embeddings/monolingual/ast_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 481846
+  "vocab_size": 477621
 }

models/subword_markov/ast_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b9be5fbc395f6cb378f05193e1ba7234945d9bca4e71b4cdad6aea648215809
-size 555949
+oid sha256:cb148bb49ec5b1e60c50c2b01fac8cc9efe43fa2426141589a2cf61214e447d5
+size 554253

models/subword_markov/ast_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "ast",
-  "unique_contexts": 10463,
-  "total_transitions": 464881103
+  "unique_contexts": 10438,
+  "total_transitions": 460362172
 }

models/subword_markov/ast_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a0a7c5f99ccca8b527cebd7afb3189b4a99a6cc219cd1b861b501d17180c93b0
-size 2891072
+oid sha256:d3f62c27dcd1d6690cf3f120e2f6a5c6224a17e0b6fff5b98a2c7fd95b82a98d
+size 2877694

models/subword_markov/ast_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "ast",
-  "unique_contexts": 83437,
-  "total_transitions": 464744539
+  "unique_contexts": 83238,
+  "total_transitions": 460224997
 }

models/subword_markov/ast_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c060d1174629162cb8911f78054119018ce6e2fc628936bfdeb36691d35489c
-size 11949355
+oid sha256:77c6f32d39e8a6d9e1e19e90d355f3621fa0210f657c9f5552ad05bd47ee32b3
+size 11877512

models/subword_markov/ast_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "ast",
-  "unique_contexts": 357207,
-  "total_transitions": 464607975
+  "unique_contexts": 356042,
+  "total_transitions": 460087822
 }

models/subword_markov/ast_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:60988880010bce63ffacea78379f2d4b447c453dc44f6445a08b1d6e123f645c
-size 39925220
+oid sha256:4775656e3f03420f4fc0be3be8cec53ea2eeef71de2b991104aa1e8c1d6cf3d1
+size 39682149

models/subword_markov/ast_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "ast",
-  "unique_contexts": 1439883,
-  "total_transitions": 464471411
+  "unique_contexts": 1432910,
+  "total_transitions": 459950647
 }

models/subword_ngram/ast_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c50ba055002320ed277521dcbb25feecc7139a7b02e23e72ca8255ada347410
-size 259482
+oid sha256:cd56e2dc4be03e73c58db00b0c03abc8ff8608831df1dea23265308969403942
+size 258342

models/subword_ngram/ast_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "ast",
-  "unique_ngrams": 19069,
-  "total_ngrams": 464881103
+  "unique_ngrams": 19027,
+  "total_ngrams": 460362172
 }

models/subword_ngram/ast_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d1ee7e63a9f92e3bf668df5f64f593ca6671568eb54bc02ab9e3b50ee2ae7ae4
-size 1721257
+oid sha256:e3cd57d086dcd78e58fdd8bac38b219f34dbf57fee3c237dbbfdecb422adb23a
+size 1714771

models/subword_ngram/ast_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "ast",
-  "unique_ngrams": 139212,
-  "total_ngrams": 464744539
+  "unique_ngrams": 138526,
+  "total_ngrams": 460224997
 }

models/subword_ngram/ast_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:390181c0c7fcc8b118b108e18cb830493109f584b4b87f76850993e3e9dda0be
-size 9353492
+oid sha256:0051215fc5b8e570546b680968cec7a30ab1f1f40d6482abe8de8d64d76e87ec
+size 9315195

models/subword_ngram/ast_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "ast",
-  "unique_ngrams": 791795,
-  "total_ngrams": 464607975
+  "unique_ngrams": 787142,
+  "total_ngrams": 460087822
 }

models/subword_ngram/ast_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:871de8d8f7a57fbba09ee3607d3553722836a85cb02b7adf1d183a8d446d56bc
+size 32616953

models/subword_ngram/ast_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "ast",
+  "unique_ngrams": 2701102,
+  "total_ngrams": 459950647
+}

models/tokenizer/ast_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f5014581384d443a91ffa0f71150e3ecc09244a1056ad046785f8b8cbc4ce42c
-size 511145
+oid sha256:f71c401c7da4eae84721cbc123f1a6c235b7b1dfb462c41451d14637fbd86d10
+size 511158

models/tokenizer/ast_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:13c458e53178d158dee7ca9d93a562de7bf19975484f7e21af497697c021bb6a
-size 791500
+oid sha256:2353fc5e2eeb724a4586cd0f7378c79af690b3d40482e80b53694caf29b8817c
+size 791804

models/tokenizer/ast_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b619430c9173d8e202e7905a328c5bca3d915f85a3540adb29b3d6230f10d749
-size 1364026
+oid sha256:9d86bf05751a152a6024ebd6b9ebc7e1593e13e0b64e8fdfe5ba1d6d641c2157
+size 1362774

models/tokenizer/ast_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8172313a7a1d41e2c61618ad0fab8a1cbd16688f798c007cba80fce5c693133c
-size 374227
+oid sha256:26e667c6c5b39bfc7a35b0bf9804eeb48c0be5a5d7d5c17c20b250dc8ea4b8aa
+size 374247

models/vocabulary/ast_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:42abd03dfba1393adae66a9e07842fcb85830c5f2dbcb0e037ab56b27fbc8846
-size 8277795
+oid sha256:930ab27518f9c77e63b8bcae7de6399074a0884205bb19a395d1e16cff2aa7f6
+size 8227892

models/vocabulary/ast_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "ast",
-  "vocabulary_size": 555056,
+  "vocabulary_size": 552425,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.015923489955309757,
+    "type_token_ratio": 0.01602418347861808,
     "coverage": {
-      "top_100": 0.4129270164572954,
-      "top_1000": 0.6027767233040656,
-      "top_5000": 0.7612431178669916,
-      "top_10000": 0.8237422388332004
+      "top_100": 0.41323848400651364,
+      "top_1000": 0.6030334770795577,
+      "top_5000": 0.7613726113859179,
+      "top_10000": 0.8238610516208773
     },
-    "hapax_count": 650708,
-    "hapax_ratio": 0.5396644782892838,
-    "total_documents": 136564
+    "hapax_count": 648980,
+    "hapax_ratio": 0.5401842009979981,
+    "total_documents": 137175
   }
 }

models/word_markov/ast_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df2720891f77ec4fc87c424807076f856c9ebed7d35e8609e740c581e849ba43
-size 130011519
+oid sha256:f34a0e11a7a033e74913e1642c39e04f60936c0248ff641be2235ae59cd00cd9
+size 128939886

models/word_markov/ast_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "ast",
-  "unique_contexts": 1204316,
-  "total_transitions": 75585781
+  "unique_contexts": 1199957,
+  "total_transitions": 74837316
 }

models/word_markov/ast_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:360e77ba671d9d79ef0176080b3098535eb366bbb657d0a5d965de6cbcaaf01a
-size 443953571
+oid sha256:71190039873c4a7ec294548046388646c3565a4912b05be4866e1a392e872a8e
+size 440719621

models/word_markov/ast_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "ast",
-  "unique_contexts": 15634564,
-  "total_transitions": 75449217
+  "unique_contexts": 15504920,
+  "total_transitions": 74700141
 }