LowOnMind-300k — 296,960 params, 200M tokens fineweb-edu, val ppl 27.06

README.md

@@ -0,0 +1,61 @@
+---
+license: apache-2.0
+datasets:
+- HuggingFaceFW/fineweb-edu
+language:
+- en
+pipeline_tag: text-generation
+tags:
+- lowonmind
+- tiny-lm
+- pretrained-from-scratch
+---
+
+# LowOnMind-300k
+
+Decoder-only de **296,960 parametros**, pre-treinado do zero em
+**200M tokens** do `HuggingFaceFW/fineweb-edu` (sample-10BT).
+
+Variante extrema do [DedeProGames/DynamicMind-Mini](https://huggingface.co/DedeProGames/DynamicMind-Mini)
+(8.9M params), reduzida ~30x. O orcamento de tokens e propositalmente pequeno.
+
+## Arquitetura
+
+| | valor |
+|---|---|
+| hidden_size | 64 |
+| intermediate_size | 136 |
+| num_hidden_layers | 6 |
+| heads (q / kv) | 4 / 2 (GQA, head_dim 16) |
+| vocab_size | 1024 (byte-level BPE proprio) |
+| contexto | 512 |
+| embeddings | tied |
+
+Diferencas em relacao ao DynamicMind-Mini: **QK-Norm** por cabeca, **RoPE
+pre-computado em buffer** (com re-expansao automatica), init escalado por
+1/sqrt(2L) nas projecoes residuais, e vocabulario de 1024 — necessario
+para caber no orcamento de 300k parametros.
+
+## Resultados
+
+- val loss: **3.2982**
+- val perplexity: **27.06**
+- treino: 43 min
+
+## Uso
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+tok = AutoTokenizer.from_pretrained("DedeProGames/LowOnMind-300k")
+model = AutoModelForCausalLM.from_pretrained("DedeProGames/LowOnMind-300k", trust_remote_code=True)
+
+ids = tok("The ", return_tensors="pt").input_ids
+print(tok.decode(model.generate(ids, max_new_tokens=64, use_cache=False)[0]))
+```
+
+## Limitacoes
+
+Com ~300k parametros o modelo aprende ortografia, espacamento e n-gramas curtos,
+mas **nao produz texto coerente**. E um artefato de pesquisa sobre o limite
+inferior de escala, nao um modelo de uso geral.

config.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "LowOnMindForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_lowonmind.LowOnMindConfig",
+    "AutoModelForCausalLM": "modeling_lowonmind.LowOnMindForCausalLM"
+  },
+  "bos_token_id": 0,
+  "dtype": "float32",
+  "eos_token_id": 0,
+  "hidden_size": 64,
+  "initializer_range": 0.02,
+  "intermediate_size": 136,
+  "max_position_embeddings": 512,
+  "model_type": "lowonmind",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 6,
+  "num_key_value_heads": 2,
+  "pad_token_id": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.16.1",
+  "use_cache": false,
+  "use_qk_norm": true,
+  "vocab_size": 1024
+}

configuration_lowonmind.py

@@ -0,0 +1,54 @@
+from transformers.configuration_utils import PretrainedConfig
+
+
+class LowOnMindConfig(PretrainedConfig):
+    """Config do LowOnMind.
+
+    Derivada do DynamicMindConfig (DedeProGames/DynamicMind-Mini), com
+    `use_qk_norm` e `initializer_range` adicionais.
+    """
+
+    model_type = "lowonmind"
+
+    def __init__(
+        self,
+        vocab_size=1024,
+        hidden_size=64,
+        intermediate_size=136,
+        num_hidden_layers=6,
+        num_attention_heads=4,
+        num_key_value_heads=2,
+        max_position_embeddings=512,
+        rms_norm_eps=1e-5,
+        rope_theta=10000.0,
+        attention_dropout=0.0,
+        use_qk_norm=True,
+        initializer_range=0.02,
+        tie_word_embeddings=True,
+        use_cache=False,
+        bos_token_id=0,
+        eos_token_id=0,
+        pad_token_id=1,
+        **kwargs,
+    ):
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            pad_token_id=pad_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.max_position_embeddings = max_position_embeddings
+        self.rms_norm_eps = rms_norm_eps
+        self.rope_theta = rope_theta
+        self.attention_dropout = attention_dropout
+        self.use_qk_norm = use_qk_norm
+        self.initializer_range = initializer_range
+        self.use_cache = use_cache

generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 0,
+  "do_sample": true,
+  "eos_token_id": 0,
+  "max_new_tokens": 120,
+  "pad_token_id": 1,
+  "temperature": 0.8,
+  "top_k": 40,
+  "top_p": 0.95,
+  "transformers_version": "5.16.1",
+  "use_cache": false
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25cf653b366709d03cc717d5461ca7842f7446505ab77f1a5a8b1af16f2dfee1
+size 1194464

modeling_lowonmind.py

@@ -0,0 +1,258 @@
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from transformers.modeling_utils import PreTrainedModel
+from transformers.generation import GenerationMixin
+from transformers.modeling_outputs import CausalLMOutput
+
+try:  # carregado como remote code (pacote)
+    from .configuration_lowonmind import LowOnMindConfig
+except ImportError:  # carregado como arquivo solto no sys.path
+    from configuration_lowonmind import LowOnMindConfig
+
+
+class LowOnMindRMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-5):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.eps = eps
+
+    def forward(self, x):
+        dtype = x.dtype
+        x = x.float()
+        var = x.pow(2).mean(dim=-1, keepdim=True)
+        x = x * torch.rsqrt(var + self.eps)
+        return (self.weight * x).to(dtype)
+
+
+class LowOnMindRotary(nn.Module):
+    """RoPE com cos/sin pre-computados e cacheados.
+
+    O DynamicMind-Mini recalculava inv_freq/cos/sin a cada forward. Aqui o cache
+    e construido uma vez e reaproveitado; se aparecer uma sequencia mais longa
+    (ou outro device) ele e reconstruido em vez de estourar num broadcast error.
+
+    O cache NAO e um buffer registrado de proposito: buffers nao-persistentes
+    criados no __init__ sao materializados com lixo/NaN pelo carregamento em
+    meta-device do from_pretrained. Como atributo simples ele e sempre
+    reconstruido no primeiro forward.
+    """
+
+    def __init__(self, head_dim, max_position_embeddings, base):
+        super().__init__()
+        self.head_dim = head_dim
+        self.base = base
+        self.max_position_embeddings = max_position_embeddings
+        self._cos = None
+        self._sin = None
+        self._cached_len = 0
+
+    def _build(self, seq_len, device):
+        inv_freq = 1.0 / (
+            self.base
+            ** (torch.arange(0, self.head_dim, 2, device=device, dtype=torch.float32) / self.head_dim)
+        )
+        t = torch.arange(seq_len, device=device, dtype=torch.float32)
+        freqs = torch.outer(t, inv_freq)
+        self._cos = freqs.cos()[None, None]
+        self._sin = freqs.sin()[None, None]
+        self._cached_len = seq_len
+
+    def forward(self, seq_len, dtype, device):
+        if self._cos is None or seq_len > self._cached_len or self._cos.device != device:
+            self._build(max(seq_len, self.max_position_embeddings, self._cached_len), device)
+        return self._cos[:, :, :seq_len].to(dtype), self._sin[:, :, :seq_len].to(dtype)
+
+
+def apply_rope(x, cos, sin):
+    # x: [B, H, T, head_dim]; cos/sin: [1, 1, T, head_dim // 2]
+    x_even, x_odd = x[..., 0::2], x[..., 1::2]
+    out_even = x_even * cos - x_odd * sin
+    out_odd = x_even * sin + x_odd * cos
+    return torch.stack((out_even, out_odd), dim=-1).flatten(-2)
+
+
+class LowOnMindAttention(nn.Module):
+    def __init__(self, config, rotary):
+        super().__init__()
+
+        self.hidden_size = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.num_kv_heads = config.num_key_value_heads
+        self.head_dim = config.hidden_size // config.num_attention_heads
+        self.attention_dropout = config.attention_dropout
+        self.rotary = rotary
+
+        assert self.hidden_size % self.num_heads == 0
+        assert self.num_heads % self.num_kv_heads == 0
+        assert self.head_dim % 2 == 0
+
+        self.q_proj = nn.Linear(config.hidden_size, self.num_heads * self.head_dim, bias=False)
+        self.k_proj = nn.Linear(config.hidden_size, self.num_kv_heads * self.head_dim, bias=False)
+        self.v_proj = nn.Linear(config.hidden_size, self.num_kv_heads * self.head_dim, bias=False)
+        self.o_proj = nn.Linear(self.num_heads * self.head_dim, config.hidden_size, bias=False)
+        self.o_proj._is_residual_proj = True
+
+        if config.use_qk_norm:
+            self.q_norm = LowOnMindRMSNorm(self.head_dim, config.rms_norm_eps)
+            self.k_norm = LowOnMindRMSNorm(self.head_dim, config.rms_norm_eps)
+        else:
+            self.q_norm = None
+            self.k_norm = None
+
+    def forward(self, x):
+        bsz, seq_len, _ = x.shape
+
+        q = self.q_proj(x).view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
+        k = self.k_proj(x).view(bsz, seq_len, self.num_kv_heads, self.head_dim).transpose(1, 2)
+        v = self.v_proj(x).view(bsz, seq_len, self.num_kv_heads, self.head_dim).transpose(1, 2)
+
+        if self.q_norm is not None:
+            q = self.q_norm(q)
+            k = self.k_norm(k)
+
+        cos, sin = self.rotary(seq_len, q.dtype, q.device)
+        q = apply_rope(q, cos, sin)
+        k = apply_rope(k, cos, sin)
+
+        if self.num_kv_heads != self.num_heads:
+            repeats = self.num_heads // self.num_kv_heads
+            k = k.repeat_interleave(repeats, dim=1)
+            v = v.repeat_interleave(repeats, dim=1)
+
+        y = F.scaled_dot_product_attention(
+            q,
+            k,
+            v,
+            attn_mask=None,
+            dropout_p=self.attention_dropout if self.training else 0.0,
+            is_causal=True,
+        )
+
+        y = y.transpose(1, 2).contiguous().view(bsz, seq_len, -1)
+        return self.o_proj(y)
+
+
+class LowOnMindMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
+        self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
+        self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
+        self.down_proj._is_residual_proj = True
+
+    def forward(self, x):
+        return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
+
+
+class LowOnMindBlock(nn.Module):
+    def __init__(self, config, rotary):
+        super().__init__()
+        self.input_layernorm = LowOnMindRMSNorm(config.hidden_size, config.rms_norm_eps)
+        self.self_attn = LowOnMindAttention(config, rotary)
+        self.post_attention_layernorm = LowOnMindRMSNorm(config.hidden_size, config.rms_norm_eps)
+        self.mlp = LowOnMindMLP(config)
+
+    def forward(self, x):
+        x = x + self.self_attn(self.input_layernorm(x))
+        x = x + self.mlp(self.post_attention_layernorm(x))
+        return x
+
+
+class LowOnMindPreTrainedModel(PreTrainedModel):
+    config_class = LowOnMindConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = False
+    _no_split_modules = ["LowOnMindBlock"]
+
+    def _init_weights(self, module):
+        std = self.config.initializer_range
+        if isinstance(module, nn.Linear):
+            # projecoes que escrevem no residual: init escalado por 1/sqrt(2L)
+            if getattr(module, "_is_residual_proj", False):
+                std = std / math.sqrt(2 * self.config.num_hidden_layers)
+            nn.init.normal_(module.weight, mean=0.0, std=std)
+            if module.bias is not None:
+                nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            nn.init.normal_(module.weight, mean=0.0, std=std)
+        elif isinstance(module, LowOnMindRMSNorm):
+            nn.init.ones_(module.weight)
+
+
+class LowOnMindForCausalLM(LowOnMindPreTrainedModel, GenerationMixin):
+    _tied_weights_keys = {"lm_head.weight": "embed_tokens.weight"}
+    _keys_to_ignore_on_load_missing = [r"lm_head.weight"]
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        head_dim = config.hidden_size // config.num_attention_heads
+        self.rotary = LowOnMindRotary(head_dim, config.max_position_embeddings, config.rope_theta)
+
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
+        self.layers = nn.ModuleList(
+            [LowOnMindBlock(config, self.rotary) for _ in range(config.num_hidden_layers)]
+        )
+        self.norm = LowOnMindRMSNorm(config.hidden_size, config.rms_norm_eps)
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        if config.tie_word_embeddings:
+            self.lm_head.weight = self.embed_tokens.weight
+
+        self.post_init()
+
+    def tie_weights(self, *args, **kwargs):
+        if getattr(self.config, "tie_word_embeddings", True):
+            self.lm_head.weight = self.embed_tokens.weight
+
+    def get_input_embeddings(self):
+        return self.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, value):
+        self.lm_head = value
+
+    def forward(self, input_ids=None, labels=None, **kwargs):
+        x = self.embed_tokens(input_ids)
+
+        for layer in self.layers:
+            x = layer(x)
+
+        x = self.norm(x)
+        logits = self.lm_head(x)
+
+        loss = None
+        if labels is not None:
+            shift_logits = logits[:, :-1, :].contiguous()
+            shift_labels = labels[:, 1:].contiguous()
+            loss = F.cross_entropy(
+                shift_logits.view(-1, shift_logits.size(-1)),
+                shift_labels.view(-1),
+                ignore_index=-100,
+            )
+
+        return CausalLMOutput(loss=loss, logits=logits)
+
+    def state_dict(self, *args, **kwargs):
+        sd = super().state_dict(*args, **kwargs)
+        # lm_head.weight e tied com embed_tokens.weight; safetensors nao guarda
+        # tensores compartilhados duplicados.
+        if getattr(self.config, "tie_word_embeddings", True):
+            for k in list(sd.keys()):
+                if k == "lm_head.weight" or k.endswith(".lm_head.weight"):
+                    del sd[k]
+        return sd
+
+    def prepare_inputs_for_generation(self, input_ids, **kwargs):
+        # sem KV cache: a janela e truncada em max_position_embeddings
+        input_ids = input_ids[:, -self.config.max_position_embeddings:]
+        return {"input_ids": input_ids}

tokenizer.json

@@ -0,0 +1,4171 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "<|pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "<|unk|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {}
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "<|endoftext|>": 0,
+      "<|pad|>": 1,
+      "<|unk|>": 2,
+      "!": 3,
+      "\"": 4,
+      "#": 5,
+      "$": 6,
+      "%": 7,
+      "&": 8,
+      "'": 9,
+      "(": 10,
+      ")": 11,
+      "*": 12,
+      "+": 13,
+      ",": 14,
+      "-": 15,
+      ".": 16,
+      "/": 17,
+      "0": 18,
+      "1": 19,
+      "2": 20,
+      "3": 21,
+      "4": 22,
+      "5": 23,
+      "6": 24,
+      "7": 25,
+      "8": 26,
+      "9": 27,
+      ":": 28,
+      ";": 29,
+      "<": 30,
+      "=": 31,
+      ">": 32,
+      "?": 33,
+      "@": 34,
+      "A": 35,
+      "B": 36,
+      "C": 37,
+      "D": 38,
+      "E": 39,
+      "F": 40,
+      "G": 41,
+      "H": 42,
+      "I": 43,
+      "J": 44,
+      "K": 45,
+      "L": 46,
+      "M": 47,
+      "N": 48,
+      "O": 49,
+      "P": 50,
+      "Q": 51,
+      "R": 52,
+      "S": 53,
+      "T": 54,
+      "U": 55,
+      "V": 56,
+      "W": 57,
+      "X": 58,
+      "Y": 59,
+      "Z": 60,
+      "[": 61,
+      "\\": 62,
+      "]": 63,
+      "^": 64,
+      "_": 65,
+      "`": 66,
+      "a": 67,
+      "b": 68,
+      "c": 69,
+      "d": 70,
+      "e": 71,
+      "f": 72,
+      "g": 73,
+      "h": 74,
+      "i": 75,
+      "j": 76,
+      "k": 77,
+      "l": 78,
+      "m": 79,
+      "n": 80,
+      "o": 81,
+      "p": 82,
+      "q": 83,
+      "r": 84,
+      "s": 85,
+      "t": 86,
+      "u": 87,
+      "v": 88,
+      "w": 89,
+      "x": 90,
+      "y": 91,
+      "z": 92,
+      "{": 93,
+      "|": 94,
+      "}": 95,
+      "~": 96,
+      "¡": 97,
+      "¢": 98,
+      "£": 99,
+      "¤": 100,
+      "¥": 101,
+      "¦": 102,
+      "§": 103,
+      "¨": 104,
+      "©": 105,
+      "ª": 106,
+      "«": 107,
+      "¬": 108,
+      "®": 109,
+      "¯": 110,
+      "°": 111,
+      "±": 112,
+      "²": 113,
+      "³": 114,
+      "´": 115,
+      "µ": 116,
+      "¶": 117,
+      "·": 118,
+      "¸": 119,
+      "¹": 120,
+      "º": 121,
+      "»": 122,
+      "¼": 123,
+      "½": 124,
+      "¾": 125,
+      "¿": 126,
+      "À": 127,
+      "Á": 128,
+      "Â": 129,
+      "Ã": 130,
+      "Ä": 131,
+      "Å": 132,
+      "Æ": 133,
+      "Ç": 134,
+      "È": 135,
+      "É": 136,
+      "Ê": 137,
+      "Ë": 138,
+      "Ì": 139,
+      "Í": 140,
+      "Î": 141,
+      "Ï": 142,
+      "Ð": 143,
+      "Ñ": 144,
+      "Ò": 145,
+      "Ó": 146,
+      "Ô": 147,
+      "Õ": 148,
+      "Ö": 149,
+      "×": 150,
+      "Ø": 151,
+      "Ù": 152,
+      "Ú": 153,
+      "Û": 154,
+      "Ü": 155,
+      "Ý": 156,
+      "Þ": 157,
+      "ß": 158,
+      "à": 159,
+      "á": 160,
+      "â": 161,
+      "ã": 162,
+      "ä": 163,
+      "å": 164,
+      "æ": 165,
+      "ç": 166,
+      "è": 167,
+      "é": 168,
+      "ê": 169,
+      "ë": 170,
+      "ì": 171,
+      "í": 172,
+      "î": 173,
+      "ï": 174,
+      "ð": 175,
+      "ñ": 176,
+      "ò": 177,
+      "ó": 178,
+      "ô": 179,
+      "õ": 180,
+      "ö": 181,
+      "÷": 182,
+      "ø": 183,
+      "ù": 184,
+      "ú": 185,
+      "û": 186,
+      "ü": 187,
+      "ý": 188,
+      "þ": 189,
+      "ÿ": 190,
+      "Ā": 191,
+      "ā": 192,
+      "Ă": 193,
+      "ă": 194,
+      "Ą": 195,
+      "ą": 196,
+      "Ć": 197,
+      "ć": 198,
+      "Ĉ": 199,
+      "ĉ": 200,
+      "Ċ": 201,
+      "ċ": 202,
+      "Č": 203,
+      "č": 204,
+      "Ď": 205,
+      "ď": 206,
+      "Đ": 207,
+      "đ": 208,
+      "Ē": 209,
+      "ē": 210,
+      "Ĕ": 211,
+      "ĕ": 212,
+      "Ė": 213,
+      "ė": 214,
+      "Ę": 215,
+      "ę": 216,
+      "Ě": 217,
+      "ě": 218,
+      "Ĝ": 219,
+      "ĝ": 220,
+      "Ğ": 221,
+      "ğ": 222,
+      "Ġ": 223,
+      "ġ": 224,
+      "Ģ": 225,
+      "ģ": 226,
+      "Ĥ": 227,
+      "ĥ": 228,
+      "Ħ": 229,
+      "ħ": 230,
+      "Ĩ": 231,
+      "ĩ": 232,
+      "Ī": 233,
+      "ī": 234,
+      "Ĭ": 235,
+      "ĭ": 236,
+      "Į": 237,
+      "į": 238,
+      "İ": 239,
+      "ı": 240,
+      "IJ": 241,
+      "ij": 242,
+      "Ĵ": 243,
+      "ĵ": 244,
+      "Ķ": 245,
+      "ķ": 246,
+      "ĸ": 247,
+      "Ĺ": 248,
+      "ĺ": 249,
+      "Ļ": 250,
+      "ļ": 251,
+      "Ľ": 252,
+      "ľ": 253,
+      "Ŀ": 254,
+      "ŀ": 255,
+      "Ł": 256,
+      "ł": 257,
+      "Ń": 258,
+      "Ġt": 259,
+      "Ġa": 260,
+      "he": 261,
+      "in": 262,
+      "re": 263,
+      "Ġthe": 264,
+      "on": 265,
+      "er": 266,
+      "Ġo": 267,
+      "at": 268,
+      "Ġs": 269,
+      "en": 270,
+      "Ġc": 271,
+      "Ġw": 272,
+      "es": 273,
+      "is": 274,
+      "nd": 275,
+      "it": 276,
+      "or": 277,
+      "Ġp": 278,
+      "al": 279,
+      "ed": 280,
+      "Ġof": 281,
+      "Ġb": 282,
+      "an": 283,
+      "Ġf": 284,
+      "ing": 285,
+      "Ġin": 286,
+      "ar": 287,
+      "ou": 288,
+      "Ġm": 289,
+      "Ġand": 290,
+      "Ġto": 291,
+      "ic": 292,
+      "Ġd": 293,
+      "ion": 294,
+      "le": 295,
+      "ro": 296,
+      "as": 297,
+      "Ġh": 298,
+      "Ġth": 299,
+      "ent": 300,
+      "ct": 301,
+      "Ġe": 302,
+      "il": 303,
+      "Ġre": 304,
+      "om": 305,
+      "ve": 306,
+      "Ġl": 307,
+      "Ġn": 308,
+      "st": 309,
+      "ly": 310,
+      "Ġbe": 311,
+      "âĢ": 312,
+      "Ġis": 313,
+      "ĠT": 314,
+      "se": 315,
+      "ol": 316,
+      "Ġg": 317,
+      "ation": 318,
+      "im": 319,
+      "id": 320,
+      "et": 321,
+      "ut": 322,
+      "ce": 323,
+      "ot": 324,
+      "ur": 325,
+      "ra": 326,
+      "ch": 327,
+      "ow": 328,
+      "ĠA": 329,
+      "Ġfor": 330,
+      "ig": 331,
+      "ĠS": 332,
+      "Ġthat": 333,
+      "Ġon": 334,
+      "Ġu": 335,
+      "Ġst": 336,
+      "ver": 337,
+      "ĠC": 338,
+      "ĠI": 339,
+      "ir": 340,
+      "Ġy": 341,
+      "ul": 342,
+      "ay": 343,
+      "am": 344,
+      "Ġas": 345,
+      "el": 346,
+      "ith": 347,
+      "ad": 348,
+      "her": 349,
+      "Ġpro": 350,
+      "Ġcon": 351,
+      "Ġare": 352,
+      "Ġan": 353,
+      "Ġwith": 354,
+      "Ġwh": 355,
+      "if": 356,
+      "Ġit": 357,
+      "Ġor": 358,
+      "Ġal": 359,
+      "ter": 360,
+      "ĠM": 361,
+      "Ġyou": 362,
+      "od": 363,
+      "Ġ(": 364,
+      "ĠThe": 365,
+      "Ġ1": 366,
+      "ĠP": 367,
+      "ge": 368,
+      "ment": 369,
+      "th": 370,
+      "ist": 371,
+      "op": 372,
+      "ate": 373,
+      "ers": 374,
+      "Ġde": 375,
+      "ies": 376,
+      "Ġhe": 377,
+      "ill": 378,
+      "ĠB": 379,
+      "ab": 380,
+      "us": 381,
+      "ess": 382,
+      "Ġex": 383,
+      "Ġwe": 384,
+      "ore": 385,
+      "ect": 386,
+      "Ġsu": 387,
+      "Ġcom": 388,
+      "ity": 389,
+      "âĢĻ": 390,
+      "est": 391,
+      "Ġha": 392,
+      "res": 393,
+      "ive": 394,
+      "ĠH": 395,
+      "ld": 396,
+      "rom": 397,
+      "um": 398,
+      "ac": 399,
+      "os": 400,
+      "Ġv": 401,
+      "Ġby": 402,
+      "ke": 403,
+      "em": 404,
+      "ant": 405,
+      "iv": 406,
+      "qu": 407,
+      "ĠW": 408,
+      "Ġat": 409,
+      "ain": 410,
+      "nt": 411,
+      "Ġwas": 412,
+      "pp": 413,
+      "ĠD": 414,
+      "igh": 415,
+      "ĠE": 416,
+      "Ġfrom": 417,
+      "ud": 418,
+      "and": 419,
+      "oc": 420,
+      "Ġch": 421,
+      "ĠR": 422,
+      "Ġcan": 423,
+      "Ġr": 424,
+      "ort": 425,
+      "Ġne": 426,
+      "ĠF": 427,
+      "Ġse": 428,
+      "ial": 429,
+      "Ġ2": 430,
+      "ure": 431,
+      "Ġnot": 432,
+      "ĠâĢ": 433,
+      "un": 434,
+      "ĠN": 435,
+      "art": 436,
+      "ĠG": 437,
+      "Ġle": 438,
+      "ical": 439,
+      "Ġhave": 440,
+      "ri": 441,
+      "ĠL": 442,
+      "og": 443,
+      "Ġsh": 444,
+      "Ġen": 445,
+      "Ġthis": 446,
+      "Ġsp": 447,
+      "00": 448,
+      "ight": 449,
+      "ich": 450,
+      "Ġab": 451,
+      "pt": 452,
+      "rou": 453,
+      "red": 454,
+      "Ġwor": 455,
+      "all": 456,
+      "ould": 457,
+      "ther": 458,
+      "gh": 459,
+      "The": 460,
+      "ome": 461,
+      "ost": 462,
+      "ions": 463,
+      "our": 464,
+      "ated": 465,
+      "du": 466,
+      "Ġim": 467,
+      "out": 468,
+      "per": 469,
+      "ine": 470,
+      "Ġus": 471,
+      "ard": 472,
+      "ag": 473,
+      "Ġpl": 474,
+      "Ġtheir": 475,
+      "ell": 476,
+      "ĠO": 477,
+      "ide": 478,
+      "ff": 479,
+      "act": 480,
+      "ans": 481,
+      "Ġint": 482,
+      "ear": 483,
+      "ast": 484,
+      "pl": 485,
+      "ĠJ": 486,
+      "age": 487,
+      "ust": 488,
+      "Ġwhich": 489,
+      "Ġk": 490,
+      "ak": 491,
+      "ous": 492,
+      "Ġthey": 493,
+      "Ġwhe": 494,
+      "ec": 495,
+      "ĠU": 496,
+      "Ġall": 497,
+      "iz": 498,
+      "ĠIn": 499,
+      "Ġcont": 500,
+      "ood": 501,
+      "Ġyour": 502,
+      "ap": 503,
+      "Ġad": 504,
+      "ib": 505,
+      "ult": 506,
+      "Ġinc": 507,
+      "ia": 508,
+      "Ġwill": 509,
+      "cc": 510,
+      "end": 511,
+      "Ġdis": 512,
+      "ru": 513,
+      "âĢĿ": 514,
+      "ong": 515,
+      "Ġmore": 516,
+      "Ġcomp": 517,
+      "ew": 518,
+      "Ġhas": 519,
+      "ip": 520,
+      "ations": 521,
+      "vel": 522,
+      "Ġres": 523,
+      "ere": 524,
+      "ary": 525,
+      "orm": 526,
+      "so": 527,
+      "Ġdo": 528,
+      "ater": 529,
+      "Ġwere": 530,
+      "ĠTh": 531,
+      "Ġother": 532,
+      "Ġbut": 533,
+      "Ġcl": 534,
+      "ild": 535,
+      "ence": 536,
+      "Ġman": 537,
+      "ual": 538,
+      "are": 539,
+      "able": 540,
+      "ks": 541,
+      "'s": 542,
+      "ry": 543,
+      "Ġper": 544,
+      "Ġone": 545,
+      "ance": 546,
+      "ĠâĢľ": 547,
+      "ind": 548,
+      "Ġ19": 549,
+      "ep": 550,
+      "ach": 551,
+      "Ġme": 552,
+      "Ġalso": 553,
+      "ents": 554,
+      "Ġimp": 555,
+      "Ġte": 556,
+      "ose": 557,
+      "reat": 558,
+      "ass": 559,
+      "ake": 560,
+      "ign": 561,
+      "ress": 562,
+      "Ġj": 563,
+      "ase": 564,
+      "au": 565,
+      "ition": 566,
+      "av": 567,
+      "ber": 568,
+      "Ġso": 569,
+      "Ġar": 570,
+      "ice": 571,
+      "Ġun": 572,
+      "ction": 573,
+      "Ġout": 574,
+      "Ġpart": 575,
+      "very": 576,
+      "Ġapp": 577,
+      "ime": 578,
+      "ish": 579,
+      "Ġhis": 580,
+      "Ġabout": 581,
+      "ord": 582,
+      "ory": 583,
+      "Ġev": 584,
+      "one": 585,
+      "ok": 586,
+      "ame": 587,
+      "Ġwho": 588,
+      "ount": 589,
+      "ens": 590,
+      "ace": 591,
+      "ĠSt": 592,
+      ").": 593,
+      "ire": 594,
+      "Ġstud": 595,
+      "ub": 596,
+      "Ġ\"": 597,
+      "Ġinf": 598,
+      "ack": 599,
+      "ĠK": 600,
+      "Ġpe": 601,
+      "Ġsc": 602,
+      "Ġthem": 603,
+      "ue": 604,
+      "ite": 605,
+      "ors": 606,
+      "Ġsy": 607,
+      "Ġcomm": 608,
+      "ree": 609,
+      "ang": 610,
+      "ound": 611,
+      "now": 612,
+      "ces": 613,
+      "ike": 614,
+      "Ġup": 615,
+      "low": 616,
+      "Ġbec": 617,
+      "ĠIt": 618,
+      "olog": 619,
+      "Ġpre": 620,
+      "ĠCh": 621,
+      "Ġthan": 622,
+      "ie": 623,
+      "ian": 624,
+      "ile": 625,
+      "Ġmay": 626,
+      "Ġyear": 627,
+      "ric": 628,
+      "Ġqu": 629,
+      "Ġsome": 630,
+      "iff": 631,
+      "Ġbeen": 632,
+      "ays": 633,
+      "Ġthere": 634,
+      "ates": 635,
+      "Ġrec": 636,
+      "ople": 637,
+      "ail": 638,
+      "ings": 639,
+      "nder": 640,
+      "mer": 641,
+      "Ġwhen": 642,
+      "ough": 643,
+      "Ġits": 644,
+      "Ġag": 645,
+      "ph": 646,
+      "Ġinto": 647,
+      "Ġgo": 648,
+      "Ġtime": 649,
+      "ren": 650,
+      "rit": 651,
+      "vers": 652,
+      "Ġtra": 653,
+      "Ġhad": 654,
+      "Ġwork": 655,
+      "ĠV": 656,
+      "ons": 657,
+      "Ġ20": 658,
+      "erv": 659,
+      "te": 660,
+      "Ġthese": 661,
+      "rough": 662,
+      "Ġover": 663,
+      "ally": 664,
+      "ĠThis": 665,
+      "ond": 666,
+      "lud": 667,
+      "ove": 668,
+      "Ġhow": 669,
+      "Ġpeople": 670,
+      "Ġdiff": 671,
+      "Ġra": 672,
+      "ms": 673,
+      "tern": 674,
+      "Ġcons": 675,
+      "ause": 676,
+      "ll": 677,
+      "Ġcol": 678,
+      "Ġ3": 679,
+      "Ġro": 680,
+      "Ġour": 681,
+      "Ġtr": 682,
+      "int": 683,
+      "Ġact": 684,
+      "Ġmost": 685,
+      "ific": 686,
+      "Ġbet": 687,
+      "Ġuse": 688,
+      "lic": 689,
+      "we": 690,
+      "Ġwould": 691,
+      "irst": 692,
+      "Ġknow": 693,
+      "Ġsa": 694,
+      "lect": 695,
+      "ject": 696,
+      "Ġdes": 697,
+      "ys": 698,
+      "ments": 699,
+      "Ġunder": 700,
+      "ious": 701,
+      "ities": 702,
+      "Ġlike": 703,
+      "ning": 704,
+      "ook": 705,
+      "Ġbl": 706,
+      "Ġtw": 707,
+      "row": 708,
+      "Ġacc": 709,
+      "Ġnew": 710,
+      "ĠY": 711,
+      "alth": 712,
+      "les": 713,
+      "stem": 714,
+      "Ġrel": 715,
+      "velop": 716,
+      "Ġsuch": 717,
+      "ational": 718,
+      "Ġrep": 719,
+      "ient": 720,
+      "Ġif": 721,
+      "Ġneed": 722,
+      "Ġno": 723,
+      "Ġany": 724,
+      "ffect": 725,
+      "hed": 726,
+      "Ġinclud": 727,
+      "Ġchild": 728,
+      "Ġgen": 729,
+      "ative": 730,
+      "Ġonly": 731,
+      "ular": 732,
+      "ple": 733,
+      "Ġwhat": 734,
+      "),": 735,
+      "Ġpr": 736,
+      "ob": 737,
+      "Ġhel": 738,
+      "ced": 739,
+      "cess": 740,
+      "Ġprov": 741,
+      "ov": 742,
+      "Ġph": 743,
+      "ĠUn": 744,
+      "Ġpo": 745,
+      "Ġfirst": 746,
+      "ating": 747,
+      "Ġmany": 748,
+      "Ġused": 749,
+      "Ġthrough": 750,
+      "Ġprodu": 751,
+      "yp": 752,
+      "Ġhelp": 753,
+      "iet": 754,
+      "Ġoff": 755,
+      "uc": 756,
+      "Ġind": 757,
+      "Ġam": 758,
+      "erm": 759,
+      "oss": 760,
+      "ath": 761,
+      "Ġdevelop": 762,
+      "ise": 763,
+      "fter": 764,
+      "round": 765,
+      "ool": 766,
+      "Ġspec": 767,
+      "ne": 768,
+      "rib": 769,
+      "Ġac": 770,
+      "Ġem": 771,
+      "ark": 772,
+      "ts": 773,
+      "Ġreg": 774,
+      "Ġtwo": 775,
+      "uring": 776,
+      "ics": 777,
+      "Ġexp": 778,
+      "Ġfe": 779,
+      "ink": 780,
+      "hen": 781,
+      "iss": 782,
+      "Ġsystem": 783,
+      "ful": 784,
+      "Ġob": 785,
+      "ied": 786,
+      "old": 787,
+      "oth": 788,
+      "eth": 789,
+      "ility": 790,
+      "Ġass": 791,
+      "Ġatt": 792,
+      "Ġdiffere": 793,
+      "ible": 794,
+      "Ġdec": 795,
+      "Ġ-": 796,
+      "Ġsur": 797,
+      "rop": 798,
+      "ife": 799,
+      "ĠHe": 800,
+      "ased": 801,
+      "ten": 802,
+      "als": 803,
+      "ert": 804,
+      "Ġ200": 805,
+      "Ġadd": 806,
+      "ng": 807,
+      "ange": 808,
+      "arch": 809,
+      "ĠâĢĵ": 810,
+      "aw": 811,
+      "ower": 812,
+      "ble": 813,
+      "fore": 814,
+      "ram": 815,
+      "In": 816,
+      "Ġ201": 817,
+      "Ġshould": 818,
+      "Ġyears": 819,
+      "ures": 820,
+      "Ġfl": 821,
+      "Ġsupp": 822,
+      "sel": 823,
+      "Ġwell": 824,
+      "ween": 825,
+      "Ġrem": 826,
+      "ract": 827,
+      "Ġinv": 828,
+      "own": 829,
+      "ade": 830,
+      "Ġco": 831,
+      "Ġ4": 832,
+      "ution": 833,
+      "Ġcould": 834,
+      "gan": 835,
+      "Ġsub": 836,
+      "its": 837,
+      "Ġwater": 838,
+      "Ġsm": 839,
+      "Ġbetween": 840,
+      "cy": 841,
+      "wn": 842,
+      "Ġwhere": 843,
+      "hes": 844,
+      "ily": 845,
+      "Ġpres": 846,
+      "Ġhigh": 847,
+      "uch": 848,
+      "Ġher": 849,
+      "Ġear": 850,
+      "land": 851,
+      "ick": 852,
+      "Ġmake": 853,
+      "Ġhealth": 854,
+      "ident": 855,
+      "ited": 856,
+      "xt": 857,
+      "Ġ5": 858,
+      "ĠAs": 859,
+      "Ġtrans": 860,
+      "ever": 861,
+      "Ġpers": 862,
+      "form": 863,
+      "Ġbu": 864,
+      "Ġrese": 865,
+      "Ġget": 866,
+      "Ġinter": 867,
+      "ology": 868,
+      "ants": 869,
+      "ivers": 870,
+      "meric": 871,
+      "Ġimport": 872,
+      "Ġdisc": 873,
+      "Ġpol": 874,
+      "ists": 875,
+      "ural": 876,
+      "Ġform": 877,
+      "Ġmod": 878,
+      "ience": 879,
+      "chool": 880,
+      "ting": 881,
+      "Ġinst": 882,
+      "Ġnum": 883,
+      "ism": 884,
+      "ines": 885,
+      "Ġfound": 886,
+      "chn": 887,
+      "ron": 888,
+      "Ġexam": 889,
+      "ict": 890,
+      "Ġmed": 891,
+      "ef": 892,
+      "Ġcent": 893,
+      "ins": 894,
+      "Ġthen": 895,
+      "gy": 896,
+      "ually": 897,
+      "Ġ18": 898,
+      "Ġlong": 899,
+      "Ġlar": 900,
+      "Ġeven": 901,
+      "Ġbecause": 902,
+      "oy": 903,
+      "Ġhum": 904,
+      "Ġlear": 905,
+      "urn": 906,
+      "Ġafter": 907,
+      "Ġworld": 908,
+      "Ġvery": 909,
+      "Ġinform": 910,
+      "Ġjust": 911,
+      "cept": 912,
+      "Ġsim": 913,
+      "Ġdifferent": 914,
+      "ature": 915,
+      "Ġed": 916,
+      "other": 917,
+      "Ġcount": 918,
+      "ft": 919,
+      "Ġincre": 920,
+      "hat": 921,
+      "Ġexper": 922,
+      "Ġeffect": 923,
+      "Ġway": 924,
+      "ness": 925,
+      "ata": 926,
+      "Ġdet": 927,
+      "ĠAmeric": 928,
+      "Ġref": 929,
+      "Ġsee": 930,
+      "Ġpub": 931,
+      "ically": 932,
+      "get": 933,
+      "Ġcar": 934,
+      "Ġfol": 935,
+      "aking": 936,
+      "Ġthose": 937,
+      "vent": 938,
+      "Ġcall": 939,
+      "ives": 940,
+      "Ġsoc": 941,
+      "Ġresp": 942,
+      "Ġgener": 943,
+      "Ġtyp": 944,
+      "tle": 945,
+      "iew": 946,
+      "Ġsaid": 947,
+      "Ġevery": 948,
+      "ork": 949,
+      "Ġvar": 950,
+      "Ġke": 951,
+      "cer": 952,
+      "Ġext": 953,
+      "omm": 954,
+      "ck": 955,
+      "Ġsec": 956,
+      "Ġprog": 957,
+      "Ġbeing": 958,
+      "Ġsign": 959,
+      "Ġchildren": 960,
+      "ds": 961,
+      "Ġloc": 962,
+      "ues": 963,
+      "con": 964,
+      "Ġfact": 965,
+      "arm": 966,
+      "Ġeach": 967,
+      "led": 968,
+      "ros": 969,
+      "ases": 970,
+      ".âĢĿ": 971,
+      "Ġset": 972,
+      "vern": 973,
+      "Ġgra": 974,
+      "Ġpartic": 975,
+      "ruct": 976,
+      "roup": 977,
+      "Ġlife": 978,
+      "Ġresult": 979,
+      "Ġshe": 980,
+      "Ġca": 981,
+      "Ġpat": 982,
+      "..": 983,
+      "ather": 984,
+      "ĠAl": 985,
+      "Ġfam": 986,
+      "Ġmy": 987,
+      "Ġdist": 988,
+      "Ġmuch": 989,
+      "Ġart": 990,
+      "cent": 991,
+      "Ġprocess": 992,
+      "Ġdef": 993,
+      "ages": 994,
+      "Ġduring": 995,
+      "ved": 996,
+      "Ġown": 997,
+      "Ġmon": 998,
+      "Ġread": 999,
+      "Th": 1000,
+      "Ġinformation": 1001,
+      "ĠWe": 1002,
+      "ody": 1003,
+      "Ġimportant": 1004,
+      "ĠPro": 1005,
+      "Ġrequ": 1006,
+      "Ġcre": 1007,
+      "ward": 1008,
+      "Ġent": 1009,
+      "Ġcommun": 1010,
+      "iron": 1011,
+      "Ġright": 1012,
+      "imes": 1013,
+      "Ġsame": 1014,
+      "Ġtechn": 1015,
+      "Ġaut": 1016,
+      "Ġdep": 1017,
+      "Ġfollow": 1018,
+      "io": 1019,
+      "Ġins": 1020,
+      "ĠThey": 1021,
+      "ized": 1022,
+      "Ġmade": 1023
+    },
+    "merges": [
+      [
+        "Ġ",
+        "t"
+      ],
+      [
+        "Ġ",
+        "a"
+      ],
+      [
+        "h",
+        "e"
+      ],
+      [
+        "i",
+        "n"
+      ],
+      [
+        "r",
+        "e"
+      ],
+      [
+        "Ġt",
+        "he"
+      ],
+      [
+        "o",
+        "n"
+      ],
+      [
+        "e",
+        "r"
+      ],
+      [
+        "Ġ",
+        "o"
+      ],
+      [
+        "a",
+        "t"
+      ],
+      [
+        "Ġ",
+        "s"
+      ],
+      [
+        "e",
+        "n"
+      ],
+      [
+        "Ġ",
+        "c"
+      ],
+      [
+        "Ġ",
+        "w"
+      ],
+      [
+        "e",
+        "s"
+      ],
+      [
+        "i",
+        "s"
+      ],
+      [
+        "n",
+        "d"
+      ],
+      [
+        "i",
+        "t"
+      ],
+      [
+        "o",
+        "r"
+      ],
+      [
+        "Ġ",
+        "p"
+      ],
+      [
+        "a",
+        "l"
+      ],
+      [
+        "e",
+        "d"
+      ],
+      [
+        "Ġo",
+        "f"
+      ],
+      [
+        "Ġ",
+        "b"
+      ],
+      [
+        "a",
+        "n"
+      ],
+      [
+        "Ġ",
+        "f"
+      ],
+      [
+        "in",
+        "g"
+      ],
+      [
+        "Ġ",
+        "in"
+      ],
+      [
+        "a",
+        "r"
+      ],
+      [
+        "o",
+        "u"
+      ],
+      [
+        "Ġ",
+        "m"
+      ],
+      [
+        "Ġa",
+        "nd"
+      ],
+      [
+        "Ġt",
+        "o"
+      ],
+      [
+        "i",
+        "c"
+      ],
+      [
+        "Ġ",
+        "d"
+      ],
+      [
+        "i",
+        "on"
+      ],
+      [
+        "l",
+        "e"
+      ],
+      [
+        "r",
+        "o"
+      ],
+      [
+        "a",
+        "s"
+      ],
+      [
+        "Ġ",
+        "h"
+      ],
+      [
+        "Ġt",
+        "h"
+      ],
+      [
+        "en",
+        "t"
+      ],
+      [
+        "c",
+        "t"
+      ],
+      [
+        "Ġ",
+        "e"
+      ],
+      [
+        "i",
+        "l"
+      ],
+      [
+        "Ġ",
+        "re"
+      ],
+      [
+        "o",
+        "m"
+      ],
+      [
+        "v",
+        "e"
+      ],
+      [
+        "Ġ",
+        "l"
+      ],
+      [
+        "Ġ",
+        "n"
+      ],
+      [
+        "s",
+        "t"
+      ],
+      [
+        "l",
+        "y"
+      ],
+      [
+        "Ġb",
+        "e"
+      ],
+      [
+        "â",
+        "Ģ"
+      ],
+      [
+        "Ġ",
+        "is"
+      ],
+      [
+        "Ġ",
+        "T"
+      ],
+      [
+        "s",
+        "e"
+      ],
+      [
+        "o",
+        "l"
+      ],
+      [
+        "Ġ",
+        "g"
+      ],
+      [
+        "at",
+        "ion"
+      ],
+      [
+        "i",
+        "m"
+      ],
+      [
+        "i",
+        "d"
+      ],
+      [
+        "e",
+        "t"
+      ],
+      [
+        "u",
+        "t"
+      ],
+      [
+        "c",
+        "e"
+      ],
+      [
+        "o",
+        "t"
+      ],
+      [
+        "u",
+        "r"
+      ],
+      [
+        "r",
+        "a"
+      ],
+      [
+        "c",
+        "h"
+      ],
+      [
+        "o",
+        "w"
+      ],
+      [
+        "Ġ",
+        "A"
+      ],
+      [
+        "Ġf",
+        "or"
+      ],
+      [
+        "i",
+        "g"
+      ],
+      [
+        "Ġ",
+        "S"
+      ],
+      [
+        "Ġth",
+        "at"
+      ],
+      [
+        "Ġ",
+        "on"
+      ],
+      [
+        "Ġ",
+        "u"
+      ],
+      [
+        "Ġs",
+        "t"
+      ],
+      [
+        "v",
+        "er"
+      ],
+      [
+        "Ġ",
+        "C"
+      ],
+      [
+        "Ġ",
+        "I"
+      ],
+      [
+        "i",
+        "r"
+      ],
+      [
+        "Ġ",
+        "y"
+      ],
+      [
+        "u",
+        "l"
+      ],
+      [
+        "a",
+        "y"
+      ],
+      [
+        "a",
+        "m"
+      ],
+      [
+        "Ġa",
+        "s"
+      ],
+      [
+        "e",
+        "l"
+      ],
+      [
+        "it",
+        "h"
+      ],
+      [
+        "a",
+        "d"
+      ],
+      [
+        "he",
+        "r"
+      ],
+      [
+        "Ġp",
+        "ro"
+      ],
+      [
+        "Ġc",
+        "on"
+      ],
+      [
+        "Ġa",
+        "re"
+      ],
+      [
+        "Ġa",
+        "n"
+      ],
+      [
+        "Ġw",
+        "ith"
+      ],
+      [
+        "Ġw",
+        "h"
+      ],
+      [
+        "i",
+        "f"
+      ],
+      [
+        "Ġ",
+        "it"
+      ],
+      [
+        "Ġo",
+        "r"
+      ],
+      [
+        "Ġa",
+        "l"
+      ],
+      [
+        "t",
+        "er"
+      ],
+      [
+        "Ġ",
+        "M"
+      ],
+      [
+        "Ġy",
+        "ou"
+      ],
+      [
+        "o",
+        "d"
+      ],
+      [
+        "Ġ",
+        "("
+      ],
+      [
+        "ĠT",
+        "he"
+      ],
+      [
+        "Ġ",
+        "1"
+      ],
+      [
+        "Ġ",
+        "P"
+      ],
+      [
+        "g",
+        "e"
+      ],
+      [
+        "m",
+        "ent"
+      ],
+      [
+        "t",
+        "h"
+      ],
+      [
+        "is",
+        "t"
+      ],
+      [
+        "o",
+        "p"
+      ],
+      [
+        "at",
+        "e"
+      ],
+      [
+        "er",
+        "s"
+      ],
+      [
+        "Ġd",
+        "e"
+      ],
+      [
+        "i",
+        "es"
+      ],
+      [
+        "Ġ",
+        "he"
+      ],
+      [
+        "il",
+        "l"
+      ],
+      [
+        "Ġ",
+        "B"
+      ],
+      [
+        "a",
+        "b"
+      ],
+      [
+        "u",
+        "s"
+      ],
+      [
+        "es",
+        "s"
+      ],
+      [
+        "Ġe",
+        "x"
+      ],
+      [
+        "Ġw",
+        "e"
+      ],
+      [
+        "o",
+        "re"
+      ],
+      [
+        "e",
+        "ct"
+      ],
+      [
+        "Ġs",
+        "u"
+      ],
+      [
+        "Ġc",
+        "om"
+      ],
+      [
+        "it",
+        "y"
+      ],
+      [
+        "âĢ",
+        "Ļ"
+      ],
+      [
+        "es",
+        "t"
+      ],
+      [
+        "Ġh",
+        "a"
+      ],
+      [
+        "re",
+        "s"
+      ],
+      [
+        "i",
+        "ve"
+      ],
+      [
+        "Ġ",
+        "H"
+      ],
+      [
+        "l",
+        "d"
+      ],
+      [
+        "ro",
+        "m"
+      ],
+      [
+        "u",
+        "m"
+      ],
+      [
+        "a",
+        "c"
+      ],
+      [
+        "o",
+        "s"
+      ],
+      [
+        "Ġ",
+        "v"
+      ],
+      [
+        "Ġb",
+        "y"
+      ],
+      [
+        "k",
+        "e"
+      ],
+      [
+        "e",
+        "m"
+      ],
+      [
+        "an",
+        "t"
+      ],
+      [
+        "i",
+        "v"
+      ],
+      [
+        "q",
+        "u"
+      ],
+      [
+        "Ġ",
+        "W"
+      ],
+      [
+        "Ġa",
+        "t"
+      ],
+      [
+        "a",
+        "in"
+      ],
+      [
+        "n",
+        "t"
+      ],
+      [
+        "Ġw",
+        "as"
+      ],
+      [
+        "p",
+        "p"
+      ],
+      [
+        "Ġ",
+        "D"
+      ],
+      [
+        "ig",
+        "h"
+      ],
+      [
+        "Ġ",
+        "E"
+      ],
+      [
+        "Ġf",
+        "rom"
+      ],
+      [
+        "u",
+        "d"
+      ],
+      [
+        "a",
+        "nd"
+      ],
+      [
+        "o",
+        "c"
+      ],
+      [
+        "Ġc",
+        "h"
+      ],
+      [
+        "Ġ",
+        "R"
+      ],
+      [
+        "Ġc",
+        "an"
+      ],
+      [
+        "Ġ",
+        "r"
+      ],
+      [
+        "or",
+        "t"
+      ],
+      [
+        "Ġn",
+        "e"
+      ],
+      [
+        "Ġ",
+        "F"
+      ],
+      [
+        "Ġs",
+        "e"
+      ],
+      [
+        "i",
+        "al"
+      ],
+      [
+        "Ġ",
+        "2"
+      ],
+      [
+        "u",
+        "re"
+      ],
+      [
+        "Ġn",
+        "ot"
+      ],
+      [
+        "Ġ",
+        "âĢ"
+      ],
+      [
+        "u",
+        "n"
+      ],
+      [
+        "Ġ",
+        "N"
+      ],
+      [
+        "ar",
+        "t"
+      ],
+      [
+        "Ġ",
+        "G"
+      ],
+      [
+        "Ġ",
+        "le"
+      ],
+      [
+        "ic",
+        "al"
+      ],
+      [
+        "Ġha",
+        "ve"
+      ],
+      [
+        "r",
+        "i"
+      ],
+      [
+        "Ġ",
+        "L"
+      ],
+      [
+        "o",
+        "g"
+      ],
+      [
+        "Ġs",
+        "h"
+      ],
+      [
+        "Ġ",
+        "en"
+      ],
+      [
+        "Ġth",
+        "is"
+      ],
+      [
+        "Ġs",
+        "p"
+      ],
+      [
+        "0",
+        "0"
+      ],
+      [
+        "igh",
+        "t"
+      ],
+      [
+        "ic",
+        "h"
+      ],
+      [
+        "Ġa",
+        "b"
+      ],
+      [
+        "p",
+        "t"
+      ],
+      [
+        "r",
+        "ou"
+      ],
+      [
+        "re",
+        "d"
+      ],
+      [
+        "Ġw",
+        "or"
+      ],
+      [
+        "al",
+        "l"
+      ],
+      [
+        "ou",
+        "ld"
+      ],
+      [
+        "t",
+        "her"
+      ],
+      [
+        "g",
+        "h"
+      ],
+      [
+        "T",
+        "he"
+      ],
+      [
+        "om",
+        "e"
+      ],
+      [
+        "o",
+        "st"
+      ],
+      [
+        "ion",
+        "s"
+      ],
+      [
+        "ou",
+        "r"
+      ],
+      [
+        "at",
+        "ed"
+      ],
+      [
+        "d",
+        "u"
+      ],
+      [
+        "Ġ",
+        "im"
+      ],
+      [
+        "ou",
+        "t"
+      ],
+      [
+        "p",
+        "er"
+      ],
+      [
+        "in",
+        "e"
+      ],
+      [
+        "Ġu",
+        "s"
+      ],
+      [
+        "ar",
+        "d"
+      ],
+      [
+        "a",
+        "g"
+      ],
+      [
+        "Ġp",
+        "l"
+      ],
+      [
+        "Ġthe",
+        "ir"
+      ],
+      [
+        "el",
+        "l"
+      ],
+      [
+        "Ġ",
+        "O"
+      ],
+      [
+        "id",
+        "e"
+      ],
+      [
+        "f",
+        "f"
+      ],
+      [
+        "a",
+        "ct"
+      ],
+      [
+        "an",
+        "s"
+      ],
+      [
+        "Ġin",
+        "t"
+      ],
+      [
+        "e",
+        "ar"
+      ],
+      [
+        "as",
+        "t"
+      ],
+      [
+        "p",
+        "l"
+      ],
+      [
+        "Ġ",
+        "J"
+      ],
+      [
+        "a",
+        "ge"
+      ],
+      [
+        "u",
+        "st"
+      ],
+      [
+        "Ġwh",
+        "ich"
+      ],
+      [
+        "Ġ",
+        "k"
+      ],
+      [
+        "a",
+        "k"
+      ],
+      [
+        "ou",
+        "s"
+      ],
+      [
+        "Ġthe",
+        "y"
+      ],
+      [
+        "Ġw",
+        "he"
+      ],
+      [
+        "e",
+        "c"
+      ],
+      [
+        "Ġ",
+        "U"
+      ],
+      [
+        "Ġal",
+        "l"
+      ],
+      [
+        "i",
+        "z"
+      ],
+      [
+        "ĠI",
+        "n"
+      ],
+      [
+        "Ġcon",
+        "t"
+      ],
+      [
+        "o",
+        "od"
+      ],
+      [
+        "Ġyou",
+        "r"
+      ],
+      [
+        "a",
+        "p"
+      ],
+      [
+        "Ġa",
+        "d"
+      ],
+      [
+        "i",
+        "b"
+      ],
+      [
+        "ul",
+        "t"
+      ],
+      [
+        "Ġin",
+        "c"
+      ],
+      [
+        "i",
+        "a"
+      ],
+      [
+        "Ġw",
+        "ill"
+      ],
+      [
+        "c",
+        "c"
+      ],
+      [
+        "en",
+        "d"
+      ],
+      [
+        "Ġd",
+        "is"
+      ],
+      [
+        "r",
+        "u"
+      ],
+      [
+        "âĢ",
+        "Ŀ"
+      ],
+      [
+        "on",
+        "g"
+      ],
+      [
+        "Ġm",
+        "ore"
+      ],
+      [
+        "Ġcom",
+        "p"
+      ],
+      [
+        "e",
+        "w"
+      ],
+      [
+        "Ġh",
+        "as"
+      ],
+      [
+        "i",
+        "p"
+      ],
+      [
+        "ation",
+        "s"
+      ],
+      [
+        "ve",
+        "l"
+      ],
+      [
+        "Ġre",
+        "s"
+      ],
+      [
+        "e",
+        "re"
+      ],
+      [
+        "ar",
+        "y"
+      ],
+      [
+        "or",
+        "m"
+      ],
+      [
+        "s",
+        "o"
+      ],
+      [
+        "Ġd",
+        "o"
+      ],
+      [
+        "at",
+        "er"
+      ],
+      [
+        "Ġwe",
+        "re"
+      ],
+      [
+        "ĠT",
+        "h"
+      ],
+      [
+        "Ġo",
+        "ther"
+      ],
+      [
+        "Ġb",
+        "ut"
+      ],
+      [
+        "Ġc",
+        "l"
+      ],
+      [
+        "il",
+        "d"
+      ],
+      [
+        "en",
+        "ce"
+      ],
+      [
+        "Ġm",
+        "an"
+      ],
+      [
+        "u",
+        "al"
+      ],
+      [
+        "a",
+        "re"
+      ],
+      [
+        "ab",
+        "le"
+      ],
+      [
+        "k",
+        "s"
+      ],
+      [
+        "'",
+        "s"
+      ],
+      [
+        "r",
+        "y"
+      ],
+      [
+        "Ġp",
+        "er"
+      ],
+      [
+        "Ġon",
+        "e"
+      ],
+      [
+        "an",
+        "ce"
+      ],
+      [
+        "ĠâĢ",
+        "ľ"
+      ],
+      [
+        "in",
+        "d"
+      ],
+      [
+        "Ġ1",
+        "9"
+      ],
+      [
+        "e",
+        "p"
+      ],
+      [
+        "a",
+        "ch"
+      ],
+      [
+        "Ġm",
+        "e"
+      ],
+      [
+        "Ġal",
+        "so"
+      ],
+      [
+        "ent",
+        "s"
+      ],
+      [
+        "Ġim",
+        "p"
+      ],
+      [
+        "Ġt",
+        "e"
+      ],
+      [
+        "o",
+        "se"
+      ],
+      [
+        "re",
+        "at"
+      ],
+      [
+        "as",
+        "s"
+      ],
+      [
+        "a",
+        "ke"
+      ],
+      [
+        "ig",
+        "n"
+      ],
+      [
+        "res",
+        "s"
+      ],
+      [
+        "Ġ",
+        "j"
+      ],
+      [
+        "as",
+        "e"
+      ],
+      [
+        "a",
+        "u"
+      ],
+      [
+        "it",
+        "ion"
+      ],
+      [
+        "a",
+        "v"
+      ],
+      [
+        "b",
+        "er"
+      ],
+      [
+        "Ġs",
+        "o"
+      ],
+      [
+        "Ġa",
+        "r"
+      ],
+      [
+        "ic",
+        "e"
+      ],
+      [
+        "Ġu",
+        "n"
+      ],
+      [
+        "ct",
+        "ion"
+      ],
+      [
+        "Ġo",
+        "ut"
+      ],
+      [
+        "Ġp",
+        "art"
+      ],
+      [
+        "ver",
+        "y"
+      ],
+      [
+        "Ġa",
+        "pp"
+      ],
+      [
+        "im",
+        "e"
+      ],
+      [
+        "is",
+        "h"
+      ],
+      [
+        "Ġh",
+        "is"
+      ],
+      [
+        "Ġab",
+        "out"
+      ],
+      [
+        "or",
+        "d"
+      ],
+      [
+        "or",
+        "y"
+      ],
+      [
+        "Ġe",
+        "v"
+      ],
+      [
+        "on",
+        "e"
+      ],
+      [
+        "o",
+        "k"
+      ],
+      [
+        "am",
+        "e"
+      ],
+      [
+        "Ġwh",
+        "o"
+      ],
+      [
+        "ou",
+        "nt"
+      ],
+      [
+        "en",
+        "s"
+      ],
+      [
+        "a",
+        "ce"
+      ],
+      [
+        "ĠS",
+        "t"
+      ],
+      [
+        ")",
+        "."
+      ],
+      [
+        "i",
+        "re"
+      ],
+      [
+        "Ġst",
+        "ud"
+      ],
+      [
+        "u",
+        "b"
+      ],
+      [
+        "Ġ",
+        "\""
+      ],
+      [
+        "Ġin",
+        "f"
+      ],
+      [
+        "ac",
+        "k"
+      ],
+      [
+        "Ġ",
+        "K"
+      ],
+      [
+        "Ġp",
+        "e"
+      ],
+      [
+        "Ġs",
+        "c"
+      ],
+      [
+        "Ġthe",
+        "m"
+      ],
+      [
+        "u",
+        "e"
+      ],
+      [
+        "it",
+        "e"
+      ],
+      [
+        "or",
+        "s"
+      ],
+      [
+        "Ġs",
+        "y"
+      ],
+      [
+        "Ġcom",
+        "m"
+      ],
+      [
+        "re",
+        "e"
+      ],
+      [
+        "an",
+        "g"
+      ],
+      [
+        "ou",
+        "nd"
+      ],
+      [
+        "n",
+        "ow"
+      ],
+      [
+        "c",
+        "es"
+      ],
+      [
+        "i",
+        "ke"
+      ],
+      [
+        "Ġu",
+        "p"
+      ],
+      [
+        "l",
+        "ow"
+      ],
+      [
+        "Ġbe",
+        "c"
+      ],
+      [
+        "ĠI",
+        "t"
+      ],
+      [
+        "ol",
+        "og"
+      ],
+      [
+        "Ġp",
+        "re"
+      ],
+      [
+        "ĠC",
+        "h"
+      ],
+      [
+        "Ġth",
+        "an"
+      ],
+      [
+        "i",
+        "e"
+      ],
+      [
+        "i",
+        "an"
+      ],
+      [
+        "i",
+        "le"
+      ],
+      [
+        "Ġm",
+        "ay"
+      ],
+      [
+        "Ġy",
+        "ear"
+      ],
+      [
+        "r",
+        "ic"
+      ],
+      [
+        "Ġ",
+        "qu"
+      ],
+      [
+        "Ġs",
+        "ome"
+      ],
+      [
+        "if",
+        "f"
+      ],
+      [
+        "Ġbe",
+        "en"
+      ],
+      [
+        "ay",
+        "s"
+      ],
+      [
+        "Ġthe",
+        "re"
+      ],
+      [
+        "at",
+        "es"
+      ],
+      [
+        "Ġre",
+        "c"
+      ],
+      [
+        "op",
+        "le"
+      ],
+      [
+        "a",
+        "il"
+      ],
+      [
+        "ing",
+        "s"
+      ],
+      [
+        "nd",
+        "er"
+      ],
+      [
+        "m",
+        "er"
+      ],
+      [
+        "Ġwhe",
+        "n"
+      ],
+      [
+        "ou",
+        "gh"
+      ],
+      [
+        "Ġit",
+        "s"
+      ],
+      [
+        "Ġa",
+        "g"
+      ],
+      [
+        "p",
+        "h"
+      ],
+      [
+        "Ġint",
+        "o"
+      ],
+      [
+        "Ġg",
+        "o"
+      ],
+      [
+        "Ġt",
+        "ime"
+      ],
+      [
+        "re",
+        "n"
+      ],
+      [
+        "r",
+        "it"
+      ],
+      [
+        "ver",
+        "s"
+      ],
+      [
+        "Ġt",
+        "ra"
+      ],
+      [
+        "Ġh",
+        "ad"
+      ],
+      [
+        "Ġwor",
+        "k"
+      ],
+      [
+        "Ġ",
+        "V"
+      ],
+      [
+        "on",
+        "s"
+      ],
+      [
+        "Ġ2",
+        "0"
+      ],
+      [
+        "er",
+        "v"
+      ],
+      [
+        "t",
+        "e"
+      ],
+      [
+        "Ġthe",
+        "se"
+      ],
+      [
+        "rou",
+        "gh"
+      ],
+      [
+        "Ġo",
+        "ver"
+      ],
+      [
+        "al",
+        "ly"
+      ],
+      [
+        "ĠTh",
+        "is"
+      ],
+      [
+        "on",
+        "d"
+      ],
+      [
+        "l",
+        "ud"
+      ],
+      [
+        "o",
+        "ve"
+      ],
+      [
+        "Ġh",
+        "ow"
+      ],
+      [
+        "Ġpe",
+        "ople"
+      ],
+      [
+        "Ġd",
+        "iff"
+      ],
+      [
+        "Ġ",
+        "ra"
+      ],
+      [
+        "m",
+        "s"
+      ],
+      [
+        "ter",
+        "n"
+      ],
+      [
+        "Ġcon",
+        "s"
+      ],
+      [
+        "au",
+        "se"
+      ],
+      [
+        "l",
+        "l"
+      ],
+      [
+        "Ġc",
+        "ol"
+      ],
+      [
+        "Ġ",
+        "3"
+      ],
+      [
+        "Ġ",
+        "ro"
+      ],
+      [
+        "Ġo",
+        "ur"
+      ],
+      [
+        "Ġt",
+        "r"
+      ],
+      [
+        "in",
+        "t"
+      ],
+      [
+        "Ġa",
+        "ct"
+      ],
+      [
+        "Ġm",
+        "ost"
+      ],
+      [
+        "if",
+        "ic"
+      ],
+      [
+        "Ġbe",
+        "t"
+      ],
+      [
+        "Ġu",
+        "se"
+      ],
+      [
+        "l",
+        "ic"
+      ],
+      [
+        "w",
+        "e"
+      ],
+      [
+        "Ġw",
+        "ould"
+      ],
+      [
+        "ir",
+        "st"
+      ],
+      [
+        "Ġk",
+        "now"
+      ],
+      [
+        "Ġs",
+        "a"
+      ],
+      [
+        "le",
+        "ct"
+      ],
+      [
+        "j",
+        "ect"
+      ],
+      [
+        "Ġd",
+        "es"
+      ],
+      [
+        "y",
+        "s"
+      ],
+      [
+        "ment",
+        "s"
+      ],
+      [
+        "Ġu",
+        "nder"
+      ],
+      [
+        "i",
+        "ous"
+      ],
+      [
+        "it",
+        "ies"
+      ],
+      [
+        "Ġl",
+        "ike"
+      ],
+      [
+        "n",
+        "ing"
+      ],
+      [
+        "o",
+        "ok"
+      ],
+      [
+        "Ġb",
+        "l"
+      ],
+      [
+        "Ġt",
+        "w"
+      ],
+      [
+        "ro",
+        "w"
+      ],
+      [
+        "Ġa",
+        "cc"
+      ],
+      [
+        "Ġne",
+        "w"
+      ],
+      [
+        "Ġ",
+        "Y"
+      ],
+      [
+        "al",
+        "th"
+      ],
+      [
+        "l",
+        "es"
+      ],
+      [
+        "st",
+        "em"
+      ],
+      [
+        "Ġre",
+        "l"
+      ],
+      [
+        "vel",
+        "op"
+      ],
+      [
+        "Ġsu",
+        "ch"
+      ],
+      [
+        "ation",
+        "al"
+      ],
+      [
+        "Ġre",
+        "p"
+      ],
+      [
+        "i",
+        "ent"
+      ],
+      [
+        "Ġ",
+        "if"
+      ],
+      [
+        "Ġne",
+        "ed"
+      ],
+      [
+        "Ġn",
+        "o"
+      ],
+      [
+        "Ġan",
+        "y"
+      ],
+      [
+        "ff",
+        "ect"
+      ],
+      [
+        "he",
+        "d"
+      ],
+      [
+        "Ġinc",
+        "lud"
+      ],
+      [
+        "Ġch",
+        "ild"
+      ],
+      [
+        "Ġg",
+        "en"
+      ],
+      [
+        "at",
+        "ive"
+      ],
+      [
+        "Ġon",
+        "ly"
+      ],
+      [
+        "ul",
+        "ar"
+      ],
+      [
+        "p",
+        "le"
+      ],
+      [
+        "Ġwh",
+        "at"
+      ],
+      [
+        ")",
+        ","
+      ],
+      [
+        "Ġp",
+        "r"
+      ],
+      [
+        "o",
+        "b"
+      ],
+      [
+        "Ġhe",
+        "l"
+      ],
+      [
+        "c",
+        "ed"
+      ],
+      [
+        "c",
+        "ess"
+      ],
+      [
+        "Ġpro",
+        "v"
+      ],
+      [
+        "o",
+        "v"
+      ],
+      [
+        "Ġp",
+        "h"
+      ],
+      [
+        "ĠU",
+        "n"
+      ],
+      [
+        "Ġp",
+        "o"
+      ],
+      [
+        "Ġf",
+        "irst"
+      ],
+      [
+        "at",
+        "ing"
+      ],
+      [
+        "Ġman",
+        "y"
+      ],
+      [
+        "Ġus",
+        "ed"
+      ],
+      [
+        "Ġth",
+        "rough"
+      ],
+      [
+        "Ġpro",
+        "du"
+      ],
+      [
+        "y",
+        "p"
+      ],
+      [
+        "Ġhel",
+        "p"
+      ],
+      [
+        "i",
+        "et"
+      ],
+      [
+        "Ġof",
+        "f"
+      ],
+      [
+        "u",
+        "c"
+      ],
+      [
+        "Ġin",
+        "d"
+      ],
+      [
+        "Ġa",
+        "m"
+      ],
+      [
+        "er",
+        "m"
+      ],
+      [
+        "os",
+        "s"
+      ],
+      [
+        "at",
+        "h"
+      ],
+      [
+        "Ġde",
+        "velop"
+      ],
+      [
+        "is",
+        "e"
+      ],
+      [
+        "f",
+        "ter"
+      ],
+      [
+        "rou",
+        "nd"
+      ],
+      [
+        "o",
+        "ol"
+      ],
+      [
+        "Ġsp",
+        "ec"
+      ],
+      [
+        "n",
+        "e"
+      ],
+      [
+        "ri",
+        "b"
+      ],
+      [
+        "Ġa",
+        "c"
+      ],
+      [
+        "Ġe",
+        "m"
+      ],
+      [
+        "ar",
+        "k"
+      ],
+      [
+        "t",
+        "s"
+      ],
+      [
+        "Ġre",
+        "g"
+      ],
+      [
+        "Ġtw",
+        "o"
+      ],
+      [
+        "ur",
+        "ing"
+      ],
+      [
+        "ic",
+        "s"
+      ],
+      [
+        "Ġex",
+        "p"
+      ],
+      [
+        "Ġf",
+        "e"
+      ],
+      [
+        "in",
+        "k"
+      ],
+      [
+        "he",
+        "n"
+      ],
+      [
+        "is",
+        "s"
+      ],
+      [
+        "Ġsy",
+        "stem"
+      ],
+      [
+        "f",
+        "ul"
+      ],
+      [
+        "Ġo",
+        "b"
+      ],
+      [
+        "i",
+        "ed"
+      ],
+      [
+        "ol",
+        "d"
+      ],
+      [
+        "ot",
+        "h"
+      ],
+      [
+        "et",
+        "h"
+      ],
+      [
+        "il",
+        "ity"
+      ],
+      [
+        "Ġas",
+        "s"
+      ],
+      [
+        "Ġat",
+        "t"
+      ],
+      [
+        "Ġdiff",
+        "ere"
+      ],
+      [
+        "ib",
+        "le"
+      ],
+      [
+        "Ġde",
+        "c"
+      ],
+      [
+        "Ġ",
+        "-"
+      ],
+      [
+        "Ġs",
+        "ur"
+      ],
+      [
+        "ro",
+        "p"
+      ],
+      [
+        "if",
+        "e"
+      ],
+      [
+        "ĠH",
+        "e"
+      ],
+      [
+        "as",
+        "ed"
+      ],
+      [
+        "t",
+        "en"
+      ],
+      [
+        "al",
+        "s"
+      ],
+      [
+        "er",
+        "t"
+      ],
+      [
+        "Ġ2",
+        "00"
+      ],
+      [
+        "Ġad",
+        "d"
+      ],
+      [
+        "n",
+        "g"
+      ],
+      [
+        "an",
+        "ge"
+      ],
+      [
+        "ar",
+        "ch"
+      ],
+      [
+        "ĠâĢ",
+        "ĵ"
+      ],
+      [
+        "a",
+        "w"
+      ],
+      [
+        "ow",
+        "er"
+      ],
+      [
+        "b",
+        "le"
+      ],
+      [
+        "f",
+        "ore"
+      ],
+      [
+        "ra",
+        "m"
+      ],
+      [
+        "I",
+        "n"
+      ],
+      [
+        "Ġ20",
+        "1"
+      ],
+      [
+        "Ġsh",
+        "ould"
+      ],
+      [
+        "Ġyear",
+        "s"
+      ],
+      [
+        "u",
+        "res"
+      ],
+      [
+        "Ġf",
+        "l"
+      ],
+      [
+        "Ġsu",
+        "pp"
+      ],
+      [
+        "se",
+        "l"
+      ],
+      [
+        "Ġw",
+        "ell"
+      ],
+      [
+        "we",
+        "en"
+      ],
+      [
+        "Ġre",
+        "m"
+      ],
+      [
+        "ra",
+        "ct"
+      ],
+      [
+        "Ġin",
+        "v"
+      ],
+      [
+        "ow",
+        "n"
+      ],
+      [
+        "ad",
+        "e"
+      ],
+      [
+        "Ġc",
+        "o"
+      ],
+      [
+        "Ġ",
+        "4"
+      ],
+      [
+        "ut",
+        "ion"
+      ],
+      [
+        "Ġc",
+        "ould"
+      ],
+      [
+        "g",
+        "an"
+      ],
+      [
+        "Ġsu",
+        "b"
+      ],
+      [
+        "it",
+        "s"
+      ],
+      [
+        "Ġw",
+        "ater"
+      ],
+      [
+        "Ġs",
+        "m"
+      ],
+      [
+        "Ġbet",
+        "ween"
+      ],
+      [
+        "c",
+        "y"
+      ],
+      [
+        "w",
+        "n"
+      ],
+      [
+        "Ġwhe",
+        "re"
+      ],
+      [
+        "he",
+        "s"
+      ],
+      [
+        "il",
+        "y"
+      ],
+      [
+        "Ġp",
+        "res"
+      ],
+      [
+        "Ġh",
+        "igh"
+      ],
+      [
+        "u",
+        "ch"
+      ],
+      [
+        "Ġ",
+        "her"
+      ],
+      [
+        "Ġe",
+        "ar"
+      ],
+      [
+        "l",
+        "and"
+      ],
+      [
+        "ic",
+        "k"
+      ],
+      [
+        "Ġm",
+        "ake"
+      ],
+      [
+        "Ġhe",
+        "alth"
+      ],
+      [
+        "id",
+        "ent"
+      ],
+      [
+        "it",
+        "ed"
+      ],
+      [
+        "x",
+        "t"
+      ],
+      [
+        "Ġ",
+        "5"
+      ],
+      [
+        "ĠA",
+        "s"
+      ],
+      [
+        "Ġtr",
+        "ans"
+      ],
+      [
+        "e",
+        "ver"
+      ],
+      [
+        "Ġp",
+        "ers"
+      ],
+      [
+        "f",
+        "orm"
+      ],
+      [
+        "Ġb",
+        "u"
+      ],
+      [
+        "Ġre",
+        "se"
+      ],
+      [
+        "Ġg",
+        "et"
+      ],
+      [
+        "Ġin",
+        "ter"
+      ],
+      [
+        "olog",
+        "y"
+      ],
+      [
+        "ant",
+        "s"
+      ],
+      [
+        "i",
+        "vers"
+      ],
+      [
+        "mer",
+        "ic"
+      ],
+      [
+        "Ġimp",
+        "ort"
+      ],
+      [
+        "Ġdis",
+        "c"
+      ],
+      [
+        "Ġp",
+        "ol"
+      ],
+      [
+        "ist",
+        "s"
+      ],
+      [
+        "ur",
+        "al"
+      ],
+      [
+        "Ġfor",
+        "m"
+      ],
+      [
+        "Ġm",
+        "od"
+      ],
+      [
+        "i",
+        "ence"
+      ],
+      [
+        "ch",
+        "ool"
+      ],
+      [
+        "t",
+        "ing"
+      ],
+      [
+        "Ġin",
+        "st"
+      ],
+      [
+        "Ġn",
+        "um"
+      ],
+      [
+        "is",
+        "m"
+      ],
+      [
+        "in",
+        "es"
+      ],
+      [
+        "Ġf",
+        "ound"
+      ],
+      [
+        "ch",
+        "n"
+      ],
+      [
+        "r",
+        "on"
+      ],
+      [
+        "Ġex",
+        "am"
+      ],
+      [
+        "ic",
+        "t"
+      ],
+      [
+        "Ġm",
+        "ed"
+      ],
+      [
+        "e",
+        "f"
+      ],
+      [
+        "Ġc",
+        "ent"
+      ],
+      [
+        "in",
+        "s"
+      ],
+      [
+        "Ġthe",
+        "n"
+      ],
+      [
+        "g",
+        "y"
+      ],
+      [
+        "ual",
+        "ly"
+      ],
+      [
+        "Ġ1",
+        "8"
+      ],
+      [
+        "Ġl",
+        "ong"
+      ],
+      [
+        "Ġl",
+        "ar"
+      ],
+      [
+        "Ġev",
+        "en"
+      ],
+      [
+        "Ġbec",
+        "ause"
+      ],
+      [
+        "o",
+        "y"
+      ],
+      [
+        "Ġh",
+        "um"
+      ],
+      [
+        "Ġle",
+        "ar"
+      ],
+      [
+        "ur",
+        "n"
+      ],
+      [
+        "Ġa",
+        "fter"
+      ],
+      [
+        "Ġwor",
+        "ld"
+      ],
+      [
+        "Ġ",
+        "very"
+      ],
+      [
+        "Ġinf",
+        "orm"
+      ],
+      [
+        "Ġj",
+        "ust"
+      ],
+      [
+        "ce",
+        "pt"
+      ],
+      [
+        "Ġs",
+        "im"
+      ],
+      [
+        "Ġdiffere",
+        "nt"
+      ],
+      [
+        "at",
+        "ure"
+      ],
+      [
+        "Ġ",
+        "ed"
+      ],
+      [
+        "ot",
+        "her"
+      ],
+      [
+        "Ġc",
+        "ount"
+      ],
+      [
+        "f",
+        "t"
+      ],
+      [
+        "Ġinc",
+        "re"
+      ],
+      [
+        "h",
+        "at"
+      ],
+      [
+        "Ġex",
+        "per"
+      ],
+      [
+        "Ġe",
+        "ffect"
+      ],
+      [
+        "Ġw",
+        "ay"
+      ],
+      [
+        "n",
+        "ess"
+      ],
+      [
+        "at",
+        "a"
+      ],
+      [
+        "Ġd",
+        "et"
+      ],
+      [
+        "ĠA",
+        "meric"
+      ],
+      [
+        "Ġre",
+        "f"
+      ],
+      [
+        "Ġse",
+        "e"
+      ],
+      [
+        "Ġp",
+        "ub"
+      ],
+      [
+        "ical",
+        "ly"
+      ],
+      [
+        "g",
+        "et"
+      ],
+      [
+        "Ġc",
+        "ar"
+      ],
+      [
+        "Ġf",
+        "ol"
+      ],
+      [
+        "ak",
+        "ing"
+      ],
+      [
+        "Ġth",
+        "ose"
+      ],
+      [
+        "v",
+        "ent"
+      ],
+      [
+        "Ġc",
+        "all"
+      ],
+      [
+        "iv",
+        "es"
+      ],
+      [
+        "Ġs",
+        "oc"
+      ],
+      [
+        "Ġres",
+        "p"
+      ],
+      [
+        "Ġgen",
+        "er"
+      ],
+      [
+        "Ġt",
+        "yp"
+      ],
+      [
+        "t",
+        "le"
+      ],
+      [
+        "i",
+        "ew"
+      ],
+      [
+        "Ġsa",
+        "id"
+      ],
+      [
+        "Ġe",
+        "very"
+      ],
+      [
+        "or",
+        "k"
+      ],
+      [
+        "Ġv",
+        "ar"
+      ],
+      [
+        "Ġ",
+        "ke"
+      ],
+      [
+        "c",
+        "er"
+      ],
+      [
+        "Ġex",
+        "t"
+      ],
+      [
+        "om",
+        "m"
+      ],
+      [
+        "c",
+        "k"
+      ],
+      [
+        "Ġse",
+        "c"
+      ],
+      [
+        "Ġpro",
+        "g"
+      ],
+      [
+        "Ġbe",
+        "ing"
+      ],
+      [
+        "Ġs",
+        "ign"
+      ],
+      [
+        "Ġchild",
+        "ren"
+      ],
+      [
+        "d",
+        "s"
+      ],
+      [
+        "Ġl",
+        "oc"
+      ],
+      [
+        "u",
+        "es"
+      ],
+      [
+        "c",
+        "on"
+      ],
+      [
+        "Ġf",
+        "act"
+      ],
+      [
+        "ar",
+        "m"
+      ],
+      [
+        "Ġe",
+        "ach"
+      ],
+      [
+        "l",
+        "ed"
+      ],
+      [
+        "ro",
+        "s"
+      ],
+      [
+        "as",
+        "es"
+      ],
+      [
+        ".",
+        "âĢĿ"
+      ],
+      [
+        "Ġs",
+        "et"
+      ],
+      [
+        "ver",
+        "n"
+      ],
+      [
+        "Ġg",
+        "ra"
+      ],
+      [
+        "Ġpart",
+        "ic"
+      ],
+      [
+        "ru",
+        "ct"
+      ],
+      [
+        "rou",
+        "p"
+      ],
+      [
+        "Ġl",
+        "ife"
+      ],
+      [
+        "Ġres",
+        "ult"
+      ],
+      [
+        "Ġs",
+        "he"
+      ],
+      [
+        "Ġc",
+        "a"
+      ],
+      [
+        "Ġp",
+        "at"
+      ],
+      [
+        ".",
+        "."
+      ],
+      [
+        "at",
+        "her"
+      ],
+      [
+        "ĠA",
+        "l"
+      ],
+      [
+        "Ġf",
+        "am"
+      ],
+      [
+        "Ġm",
+        "y"
+      ],
+      [
+        "Ġd",
+        "ist"
+      ],
+      [
+        "Ġm",
+        "uch"
+      ],
+      [
+        "Ġar",
+        "t"
+      ],
+      [
+        "c",
+        "ent"
+      ],
+      [
+        "Ġpro",
+        "cess"
+      ],
+      [
+        "Ġde",
+        "f"
+      ],
+      [
+        "ag",
+        "es"
+      ],
+      [
+        "Ġd",
+        "uring"
+      ],
+      [
+        "v",
+        "ed"
+      ],
+      [
+        "Ġo",
+        "wn"
+      ],
+      [
+        "Ġm",
+        "on"
+      ],
+      [
+        "Ġre",
+        "ad"
+      ],
+      [
+        "T",
+        "h"
+      ],
+      [
+        "Ġinform",
+        "ation"
+      ],
+      [
+        "ĠW",
+        "e"
+      ],
+      [
+        "od",
+        "y"
+      ],
+      [
+        "Ġimport",
+        "ant"
+      ],
+      [
+        "ĠP",
+        "ro"
+      ],
+      [
+        "Ġre",
+        "qu"
+      ],
+      [
+        "Ġc",
+        "re"
+      ],
+      [
+        "w",
+        "ard"
+      ],
+      [
+        "Ġ",
+        "ent"
+      ],
+      [
+        "Ġcomm",
+        "un"
+      ],
+      [
+        "ir",
+        "on"
+      ],
+      [
+        "Ġr",
+        "ight"
+      ],
+      [
+        "im",
+        "es"
+      ],
+      [
+        "Ġs",
+        "ame"
+      ],
+      [
+        "Ġte",
+        "chn"
+      ],
+      [
+        "Ġa",
+        "ut"
+      ],
+      [
+        "Ġde",
+        "p"
+      ],
+      [
+        "Ġfol",
+        "low"
+      ],
+      [
+        "i",
+        "o"
+      ],
+      [
+        "Ġin",
+        "s"
+      ],
+      [
+        "ĠThe",
+        "y"
+      ],
+      [
+        "iz",
+        "ed"
+      ],
+      [
+        "Ġm",
+        "ade"
+      ]
+    ]
+  }
+}

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 512,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|unk|>"
+}

train_args.json

@@ -0,0 +1,95 @@
+{
+  "data": "fineweb_edu_200M_v1024.uint16.bin",
+  "base": "LowOnMind-300k",
+  "derived_from": "DedeProGames/DynamicMind-Mini",
+  "dataset": "HuggingFaceFW/fineweb-edu:sample-10BT",
+  "tokenizer": "byte-level BPE 1024",
+  "params": 296960,
+  "seq_len": 512,
+  "batch_size": 64,
+  "grad_accum": 1,
+  "max_steps": 6103,
+  "total_tokens": 199983104,
+  "lr": 0.002,
+  "min_lr": 0.0002,
+  "warmup_steps": 250,
+  "weight_decay": 0.1,
+  "grad_clip": 1.0,
+  "val_tokens": 2000000,
+  "eval_every": 500,
+  "eval_batches": 40,
+  "device": "cuda",
+  "dtype": "bfloat16",
+  "compile": true,
+  "seed": 1337,
+  "train_minutes": 42.6,
+  "final_val_loss": 3.2982,
+  "final_val_ppl": 27.063,
+  "history": [
+    {
+      "step": 500,
+      "val_loss": 3.8883,
+      "val_ppl": 48.827
+    },
+    {
+      "step": 1000,
+      "val_loss": 3.621,
+      "val_ppl": 37.375
+    },
+    {
+      "step": 1500,
+      "val_loss": 3.5287,
+      "val_ppl": 34.079
+    },
+    {
+      "step": 2000,
+      "val_loss": 3.4824,
+      "val_ppl": 32.537
+    },
+    {
+      "step": 2500,
+      "val_loss": 3.4508,
+      "val_ppl": 31.524
+    },
+    {
+      "step": 3000,
+      "val_loss": 3.4226,
+      "val_ppl": 30.65
+    },
+    {
+      "step": 3500,
+      "val_loss": 3.3997,
+      "val_ppl": 29.955
+    },
+    {
+      "step": 4000,
+      "val_loss": 3.3725,
+      "val_ppl": 29.151
+    },
+    {
+      "step": 4500,
+      "val_loss": 3.3489,
+      "val_ppl": 28.471
+    },
+    {
+      "step": 5000,
+      "val_loss": 3.3408,
+      "val_ppl": 28.241
+    },
+    {
+      "step": 5500,
+      "val_loss": 3.3165,
+      "val_ppl": 27.565
+    },
+    {
+      "step": 6000,
+      "val_loss": 3.3251,
+      "val_ppl": 27.801
+    },
+    {
+      "step": 6103,
+      "val_loss": 3.3125,
+      "val_ppl": 27.454
+    }
+  ]
+}