Add Mini checkpoint

README.md

@@ -0,0 +1,41 @@
+---
+license: mit
+language:
+- en
+library_name: pytorch
+tags:
+- text-generation
+- chatbot
+---
+
+# Mini
+
+A small English chatbot trained from scratch. It is a decoder-only transformer with 6 layers, 4 attention heads, an embedding size of 192, and a context window of 512 tokens. The vocabulary is 507 words. It answers short questions it has seen in its training dialogues. It is not a general-purpose assistant.
+
+## Load
+
+`model.py`, `tokenizer.py`, and `pretrained.py` from this repo need to be on the Python path.
+
+```python
+from model import TinyGPT
+from tokenizer import Tokenizer
+
+model = TinyGPT.from_pretrained("StrongDev2024/mini", trust_remote_code=True)
+tokenizer = Tokenizer.from_pretrained("StrongDev2024/mini")
+
+ids = tokenizer.encode("<user> what is 2 + 2 <bot>")
+import torch
+out = model.generate(
+    torch.tensor([ids]),
+    max_new_tokens=40,
+    temperature=0.0,
+    stop_ids={tokenizer.token_to_id["<end>"], tokenizer.token_to_id["<user>"]},
+)
+print(tokenizer.decode(out[0, len(ids) :].tolist()))
+```
+
+The prompt is `<user> your question <bot>`. Generation stops at `<end>`.
+
+## License
+
+MIT

config.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "tiny-gpt",
+  "vocab_size": 507,
+  "block_size": 512,
+  "n_layer": 6,
+  "n_head": 4,
+  "n_embd": 192,
+  "dropout": 0.0
+}

model.py

@@ -0,0 +1,168 @@
+"""A small decoder-only transformer.
+
+It reads tokens from left to right and scores the next token.
+Causal attention means a position may look at earlier tokens only.
+"""
+
+import json
+from pathlib import Path
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from pretrained import resolve_pretrained_folder
+
+# Keys TinyGPT.__init__ accepts. config.json may also carry Hub metadata.
+CONFIG_KEYS = ("vocab_size", "block_size", "n_layer", "n_head", "n_embd", "dropout")
+
+
+class CausalSelfAttention(nn.Module):
+    def __init__(self, n_embd, n_head, block_size, dropout):
+        super().__init__()
+        self.n_head = n_head
+        self.head_dim = n_embd // n_head
+        self.qkv = nn.Linear(n_embd, 3 * n_embd)
+        self.proj = nn.Linear(n_embd, n_embd)
+        self.dropout = nn.Dropout(dropout)
+        # Lower triangle is 1: each token may attend to itself and the past.
+        mask = torch.tril(torch.ones(block_size, block_size))
+        self.register_buffer("mask", mask.view(1, 1, block_size, block_size))
+
+    def forward(self, x, return_attn=False):
+        batch, time, channels = x.shape
+        qkv = self.qkv(x)
+        query, key, value = qkv.split(channels, dim=2)
+        query = query.view(batch, time, self.n_head, self.head_dim).transpose(1, 2)
+        key = key.view(batch, time, self.n_head, self.head_dim).transpose(1, 2)
+        value = value.view(batch, time, self.n_head, self.head_dim).transpose(1, 2)
+        scores = (query @ key.transpose(-2, -1)) / (self.head_dim ** 0.5)
+        scores = scores.masked_fill(self.mask[:, :, :time, :time] == 0, float("-inf"))
+        weights = F.softmax(scores, dim=-1)
+        mixed = (self.dropout(weights) @ value).transpose(1, 2).contiguous().view(batch, time, channels)
+        out = self.dropout(self.proj(mixed))
+        if return_attn:
+            return out, weights
+        return out
+
+
+class Block(nn.Module):
+    def __init__(self, n_embd, n_head, block_size, dropout):
+        super().__init__()
+        self.ln1 = nn.LayerNorm(n_embd)
+        self.attn = CausalSelfAttention(n_embd, n_head, block_size, dropout)
+        self.ln2 = nn.LayerNorm(n_embd)
+        self.mlp = nn.Sequential(
+            nn.Linear(n_embd, 4 * n_embd),
+            nn.GELU(),
+            nn.Linear(4 * n_embd, n_embd),
+            nn.Dropout(dropout),
+        )
+
+    def forward(self, x, return_attn=False):
+        attended = self.attn(self.ln1(x), return_attn=return_attn)
+        if return_attn:
+            attended, weights = attended
+        x = x + attended
+        x = x + self.mlp(self.ln2(x))
+        if return_attn:
+            return x, weights
+        return x
+
+
+class TinyGPT(nn.Module):
+    def __init__(self, vocab_size, block_size=128, n_layer=2, n_head=4, n_embd=128, dropout=0.1):
+        super().__init__()
+        if n_embd % n_head != 0:
+            raise ValueError("n_embd must be divisible by n_head")
+        self.block_size = block_size
+        self.tok_emb = nn.Embedding(vocab_size, n_embd)
+        self.pos_emb = nn.Embedding(block_size, n_embd)
+        self.drop = nn.Dropout(dropout)
+        self.blocks = nn.ModuleList(
+            [Block(n_embd, n_head, block_size, dropout) for _ in range(n_layer)]
+        )
+        self.ln_f = nn.LayerNorm(n_embd)
+        self.head = nn.Linear(n_embd, vocab_size, bias=False)
+        self.config = {
+            "vocab_size": vocab_size,
+            "block_size": block_size,
+            "n_layer": n_layer,
+            "n_head": n_head,
+            "n_embd": n_embd,
+            "dropout": dropout,
+        }
+
+    def forward(self, idx, targets=None, return_attn=False):
+        _batch, time = idx.shape
+        positions = torch.arange(time, device=idx.device)
+        x = self.drop(self.tok_emb(idx) + self.pos_emb(positions))
+        attentions = []
+        for block in self.blocks:
+            if return_attn:
+                x, weights = block(x, return_attn=True)
+                attentions.append(weights)
+            else:
+                x = block(x)
+        logits = self.head(self.ln_f(x))
+        loss = None
+        if targets is not None:
+            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
+        if return_attn:
+            return logits, loss, attentions
+        return logits, loss
+
+    @torch.no_grad()
+    def generate(self, idx, max_new_tokens, temperature=0.0, top_k=None, stop_ids=None):
+        """Append tokens until a stop token or the length limit.
+
+        temperature 0 always picks the most likely next token.
+        """
+        stop_ids = set(stop_ids or [])
+        for _ in range(max_new_tokens):
+            idx_cond = idx[:, -self.block_size :]
+            logits, _ = self(idx_cond)
+            logits = logits[:, -1, :]
+            if temperature <= 0:
+                next_id = torch.argmax(logits, dim=-1, keepdim=True)
+            else:
+                logits = logits / max(temperature, 1e-6)
+                if top_k is not None:
+                    top_values, _ = torch.topk(logits, min(top_k, logits.size(-1)))
+                    logits = logits.masked_fill(logits < top_values[:, [-1]], float("-inf"))
+                probs = F.softmax(logits, dim=-1)
+                next_id = torch.multinomial(probs, num_samples=1)
+            idx = torch.cat([idx, next_id], dim=1)
+            if int(next_id.item()) in stop_ids:
+                break
+        return idx
+
+    def save_pretrained(self, folder):
+        """Write config.json and model.safetensors for the Hugging Face Hub."""
+        folder = Path(folder)
+        folder.mkdir(parents=True, exist_ok=True)
+        payload = {"model_type": "tiny-gpt", **self.config}
+        (folder / "config.json").write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+        from safetensors.torch import save_file
+
+        state = {name: value.detach().cpu().contiguous() for name, value in self.state_dict().items()}
+        save_file(state, str(folder / "model.safetensors"))
+
+    @classmethod
+    def from_pretrained(cls, path_or_repo, **_ignored):
+        """Load Mini from a local hub folder or a Hugging Face repo id.
+
+        trust_remote_code is accepted and ignored. Import this class from
+        model.py, then call from_pretrained with the repo id.
+        """
+        folder = resolve_pretrained_folder(path_or_repo)
+        raw = json.loads((folder / "config.json").read_text(encoding="utf-8"))
+        missing = [key for key in CONFIG_KEYS if key not in raw]
+        if missing:
+            raise FileNotFoundError(f"{folder / 'config.json'} is missing {', '.join(missing)}")
+        model = cls(**{key: raw[key] for key in CONFIG_KEYS})
+        from safetensors.torch import load_file
+
+        model.load_state_dict(load_file(str(folder / "model.safetensors")))
+        model.eval()
+        return model

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15c9c12258b2842d082ed748517d15bcbf135aca2fea082707e6a05fdf4b679e
+size 18149056

pretrained.py

@@ -0,0 +1,14 @@
+"""Find a local folder or a Hugging Face repo that holds Mini's files."""
+
+from pathlib import Path
+
+
+def resolve_pretrained_folder(path_or_repo):
+    """Return a directory with config.json. Download a Hub repo id first."""
+    folder = Path(path_or_repo)
+    if folder.is_dir():
+        return folder
+    from huggingface_hub import snapshot_download
+
+    downloaded = snapshot_download(repo_id=str(path_or_repo))
+    return Path(downloaded)

tokenizer.py

@@ -0,0 +1,81 @@
+"""Turn text into token ids and token ids back into text.
+
+The model never sees letters. It sees integers. This file builds that
+mapping from the training dialogues.
+"""
+
+import json
+import re
+from pathlib import Path
+
+from pretrained import resolve_pretrained_folder
+
+# Reserved tokens. They are written into the dialogue file and into chat prompts.
+# <pad> evens out lengths if a batch needs filler.
+# <unk> stands for a word that was not in the training file.
+SPECIAL_TOKENS = ["<pad>", "<unk>", "<user>", "<bot>", "<end>"]
+
+# Special tokens are kept whole. Other text splits into words and punctuation.
+TOKEN_RE = re.compile(r"<pad>|<unk>|<user>|<bot>|<end>|\w+|[^\w\s]", re.UNICODE)
+
+
+def tokenize(text):
+    """Split lowercased text into a list of token strings."""
+    return TOKEN_RE.findall(text.lower())
+
+
+class Tokenizer:
+    def __init__(self, token_to_id):
+        self.token_to_id = dict(token_to_id)
+        self.id_to_token = {index: token for token, index in self.token_to_id.items()}
+        self.unk_id = self.token_to_id["<unk>"]
+
+    @classmethod
+    def build(cls, text):
+        """Assign an id to every special token, then to every token in the text."""
+        token_to_id = {token: index for index, token in enumerate(SPECIAL_TOKENS)}
+        for token in tokenize(text):
+            if token not in token_to_id:
+                token_to_id[token] = len(token_to_id)
+        return cls(token_to_id)
+
+    def encode(self, text):
+        """Text to a list of ids. Unknown words become <unk>."""
+        return [self.token_to_id.get(token, self.unk_id) for token in tokenize(text)]
+
+    def decode(self, ids):
+        """Ids to a readable string. Special tokens are left out of the reply."""
+        words = []
+        for token_id in ids:
+            token = self.id_to_token.get(int(token_id), "<unk>")
+            if token in SPECIAL_TOKENS:
+                continue
+            words.append(token)
+        text = " ".join(words)
+        text = re.sub(r"\s+([.,!?;:])", r"\1", text)
+        return text.strip()
+
+    def save_pretrained(self, folder):
+        """Write vocab.json and tokenizer_config.json for the Hub."""
+        folder = Path(folder)
+        folder.mkdir(parents=True, exist_ok=True)
+        (folder / "vocab.json").write_text(
+            json.dumps(self.token_to_id, ensure_ascii=False, indent=2) + "\n",
+            encoding="utf-8",
+        )
+        meta = {
+            "tokenizer_class": "Tokenizer",
+            "lowercase": True,
+            "special_tokens": SPECIAL_TOKENS,
+            "unk_token": "<unk>",
+            "pad_token": "<pad>",
+        }
+        (folder / "tokenizer_config.json").write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8")
+
+    @classmethod
+    def from_pretrained(cls, path_or_repo, **_ignored):
+        """Load the word vocabulary from a local hub folder or a Hub repo id."""
+        folder = resolve_pretrained_folder(path_or_repo)
+        raw = json.loads((folder / "vocab.json").read_text(encoding="utf-8"))
+        token_to_id = {token: int(index) for token, index in raw.items()}
+        return cls(token_to_id)

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "tokenizer_class": "Tokenizer",
+  "lowercase": true,
+  "special_tokens": [
+    "<pad>",
+    "<unk>",
+    "<user>",
+    "<bot>",
+    "<end>"
+  ],
+  "unk_token": "<unk>",
+  "pad_token": "<pad>"
+}

vocab.json

@@ -0,0 +1,509 @@
+{
+  "<pad>": 0,
+  "<unk>": 1,
+  "<user>": 2,
+  "<bot>": 3,
+  "<end>": 4,
+  "hello": 5,
+  "hi": 6,
+  ",": 7,
+  "i": 8,
+  "am": 9,
+  "mini": 10,
+  ".": 11,
+  "what": 12,
+  "do": 13,
+  "you": 14,
+  "want": 15,
+  "to": 16,
+  "talk": 17,
+  "about": 18,
+  "?": 19,
+  "hey": 20,
+  "there": 21,
+  "good": 22,
+  "morning": 23,
+  "how": 24,
+  "can": 25,
+  "help": 26,
+  "evening": 27,
+  "is": 28,
+  "your": 29,
+  "name": 30,
+  "my": 31,
+  "who": 32,
+  "are": 33,
+  "a": 34,
+  "small": 35,
+  "chatbot": 36,
+  "trained": 37,
+  "from": 38,
+  "scratch": 39,
+  "tell": 40,
+  "me": 41,
+  "doing": 42,
+  "well": 43,
+  "today": 44,
+  "ok": 45,
+  "glad": 46,
+  "hear": 47,
+  "that": 48,
+  "fine": 49,
+  "great": 50,
+  "not": 51,
+  "bad": 52,
+  "chat": 53,
+  "simple": 54,
+  "things": 55,
+  "like": 56,
+  "greetings": 57,
+  "names": 58,
+  "and": 59,
+  "questions": 60,
+  "short": 61,
+  "question": 62,
+  "will": 63,
+  "try": 64,
+  "answer": 65,
+  "need": 66,
+  "thank": 67,
+  "welcome": 68,
+  "thanks": 69,
+  "bye": 70,
+  "goodbye": 71,
+  "later": 72,
+  "see": 73,
+  "language": 74,
+  "speak": 75,
+  "english": 76,
+  "yes": 77,
+  "made": 78,
+  "was": 79,
+  "on": 80,
+  "dialogues": 81,
+  "where": 82,
+  "come": 83,
+  "1": 84,
+  "+": 85,
+  "2": 86,
+  "4": 87,
+  "3": 88,
+  "6": 89,
+  "color": 90,
+  "the": 91,
+  "sky": 92,
+  "blue": 93,
+  "grass": 94,
+  "green": 95,
+  "snow": 96,
+  "white": 97,
+  "pizza": 98,
+  "food": 99,
+  "cats": 100,
+  "dogs": 101,
+  "favorite": 102,
+  "joke": 103,
+  "why": 104,
+  "did": 105,
+  "robot": 106,
+  "sit": 107,
+  "down": 108,
+  "it": 109,
+  "had": 110,
+  "byte": 111,
+  "eat": 112,
+  "say": 113,
+  "something": 114,
+  "fun": 115,
+  "capital": 116,
+  "of": 117,
+  "france": 118,
+  "know": 119,
+  "yet": 120,
+  "ask": 121,
+  "quantum": 122,
+  "physics": 123,
+  "history": 124,
+  "write": 125,
+  "code": 126,
+  "soon": 127,
+  "nice": 128,
+  "meet": 129,
+  "too": 130,
+  "night": 131,
+  "sleep": 132,
+  "old": 133,
+  "new": 134,
+  "so": 135,
+  "very": 136,
+  "young": 137,
+  "live": 138,
+  "inside": 139,
+  "this": 140,
+  "program": 141,
+  "computer": 142,
+  "now": 143,
+  "understand": 144,
+  "animal": 145,
+  "cat": 146,
+  "sun": 147,
+  "yellow": 148,
+  "banana": 149,
+  "many": 150,
+  "legs": 151,
+  "does": 152,
+  "dog": 153,
+  "have": 154,
+  "has": 155,
+  "four": 156,
+  "8": 157,
+  "5": 158,
+  "10": 159,
+  "20": 160,
+  "count": 161,
+  "three": 162,
+  "one": 163,
+  "two": 164,
+  "weather": 165,
+  "cannot": 166,
+  "outside": 167,
+  "raining": 168,
+  "sad": 169,
+  "sorry": 170,
+  "here": 171,
+  "happy": 172,
+  "wonderful": 173,
+  "tired": 174,
+  "should": 175,
+  "rest": 176,
+  "water": 177,
+  "clear": 178,
+  "drink": 179,
+  "hot": 180,
+  "day": 181,
+  "no": 182,
+  "but": 183,
+  "12": 184,
+  "7": 185,
+  "14": 186,
+  "an": 187,
+  "apple": 188,
+  "red": 189,
+  "days": 190,
+  "in": 191,
+  "week": 192,
+  "seven": 193,
+  "opposite": 194,
+  "cold": 195,
+  "up": 196,
+  "number": 197,
+  "story": 198,
+  "sam": 199,
+  "kite": 200,
+  "wind": 201,
+  "took": 202,
+  "into": 203,
+  "tree": 204,
+  "helped": 205,
+  "climb": 206,
+  "free": 207,
+  "they": 208,
+  "carried": 209,
+  "home": 210,
+  "before": 211,
+  "named": 212,
+  "pip": 213,
+  "walk": 214,
+  "walked": 215,
+  "river": 216,
+  "happened": 217,
+  "next": 218,
+  "mended": 219,
+  "bent": 220,
+  "corner": 221,
+  "gentle": 222,
+  "flew": 223,
+  "stayed": 224,
+  "watched": 225,
+  "window": 226,
+  "another": 227,
+  "went": 228,
+  "market": 229,
+  "bought": 230,
+  "apples": 231,
+  "desk": 232,
+  "back": 233,
+  "noon": 234,
+  "he": 235,
+  "held": 236,
+  "string": 237,
+  "with": 238,
+  "both": 239,
+  "hands": 240,
+  "ran": 241,
+  "across": 242,
+  "strong": 243,
+  "climbed": 244,
+  "over": 245,
+  "fence": 246,
+  "stood": 247,
+  "by": 248,
+  "gate": 249,
+  "dance": 250,
+  "pulled": 251,
+  "harder": 252,
+  "slipped": 253,
+  "'": 254,
+  "s": 255,
+  "hand": 256,
+  "at": 257,
+  "end": 258,
+  "field": 259,
+  "caught": 260,
+  "high": 261,
+  "branch": 262,
+  "hung": 263,
+  "thin": 264,
+  "rope": 265,
+  "stopped": 266,
+  "running": 267,
+  "looked": 268,
+  "reach": 269,
+  "lowest": 270,
+  "close": 271,
+  "ground": 272,
+  "still": 273,
+  "while": 274,
+  "moved": 275,
+  "slowly": 276,
+  "leaves": 277,
+  "brushed": 278,
+  "his": 279,
+  "face": 280,
+  "bird": 281,
+  "hopped": 282,
+  "away": 283,
+  "reached": 284,
+  "lifted": 285,
+  "paper": 286,
+  "together": 287,
+  "against": 288,
+  "chest": 289,
+  "wound": 290,
+  "neat": 291,
+  "loop": 292,
+  "sat": 293,
+  "ate": 294,
+  "each": 295,
+  "said": 296,
+  "would": 297,
+  "fly": 298,
+  "again": 299,
+  "after": 300,
+  "grew": 301,
+  "quiet": 302,
+  "along": 303,
+  "path": 304,
+  "were": 305,
+  "left": 306,
+  "barked": 307,
+  "once": 308,
+  "yard": 309,
+  "then": 310,
+  "lay": 311,
+  "lights": 312,
+  "came": 313,
+  "windows": 314,
+  "put": 315,
+  "chair": 316,
+  "door": 317,
+  "piece": 318,
+  "same": 319,
+  "time": 320,
+  "let": 321,
+  "go": 322,
+  "rose": 323,
+  "steadied": 324,
+  "air": 325,
+  "laughed": 326,
+  "counted": 327,
+  "turns": 328,
+  "she": 329,
+  "likes": 330,
+  "warm": 331,
+  "places": 332,
+  "jumps": 333,
+  "onto": 334,
+  "sits": 335,
+  "beside": 336,
+  "folds": 337,
+  "her": 338,
+  "paws": 339,
+  "shuts": 340,
+  "eyes": 341,
+  "works": 342,
+  "types": 343,
+  "lines": 344,
+  "read": 345,
+  "them": 346,
+  "listens": 347,
+  "sound": 348,
+  "keys": 349,
+  "pours": 350,
+  "bowl": 351,
+  "drinks": 352,
+  "looks": 353,
+  "birds": 354,
+  "wire": 355,
+  "ears": 356,
+  "turn": 357,
+  "toward": 358,
+  "stays": 359,
+  "knows": 360,
+  "shut": 361,
+  "safe": 362,
+  "drinking": 363,
+  "washes": 364,
+  "paw": 365,
+  "afternoon": 366,
+  "moves": 367,
+  "floor": 368,
+  "follows": 369,
+  "patch": 370,
+  "light": 371,
+  "sleeps": 372,
+  "until": 373,
+  "reaches": 374,
+  "wall": 375,
+  "walks": 376,
+  "kitchen": 377,
+  "even": 378,
+  "when": 379,
+  "full": 380,
+  "says": 381,
+  "answers": 382,
+  "move": 383,
+  "set": 384,
+  "climbs": 385,
+  "watches": 386,
+  "sometimes": 387,
+  "hiss": 388,
+  "only": 389,
+  "tail": 390,
+  "house": 391,
+  "returns": 392,
+  "curls": 393,
+  "off": 394,
+  "room": 395,
+  "goes": 396,
+  "dark": 397,
+  "sunday": 398,
+  "narrow": 399,
+  "lined": 400,
+  "tall": 401,
+  "stones": 402,
+  "as": 403,
+  "found": 404,
+  "stone": 405,
+  "gray": 406,
+  "stripe": 407,
+  "flat": 408,
+  "for": 409,
+  "skipping": 410,
+  "kept": 411,
+  "pocket": 412,
+  "slow": 413,
+  "shone": 414,
+  "touched": 415,
+  "threw": 416,
+  "skipped": 417,
+  "twice": 418,
+  "sank": 419,
+  "tried": 420,
+  "times": 421,
+  "splash": 422,
+  "duck": 423,
+  "swam": 424,
+  "past": 425,
+  "care": 426,
+  "log": 427,
+  "shared": 428,
+  "bread": 429,
+  "asked": 430,
+  "moving": 431,
+  "sea": 432,
+  "drank": 433,
+  "bottle": 434,
+  "clouds": 435,
+  "gathered": 436,
+  "rain": 437,
+  "fell": 438,
+  "faded": 439,
+  "striped": 440,
+  "nothing": 441,
+  "memory": 442,
+  "skips": 443,
+  "saw": 444,
+  "stuck": 445,
+  "empty": 446,
+  "watching": 447,
+  "opened": 448,
+  "jumped": 449,
+  "rubbed": 450,
+  "leg": 451,
+  "table": 452,
+  "told": 453,
+  "everyone": 454,
+  "monday": 455,
+  "road": 456,
+  "bright": 457,
+  "cool": 458,
+  "bag": 459,
+  "list": 460,
+  "square": 461,
+  "row": 462,
+  "tables": 463,
+  "under": 464,
+  "long": 465,
+  "cloth": 466,
+  "roof": 467,
+  "woman": 468,
+  "sold": 469,
+  "wooden": 470,
+  "box": 471,
+  "picked": 472,
+  "paid": 473,
+  "coins": 474,
+  "loaves": 475,
+  "round": 476,
+  "brown": 477,
+  "loaf": 478,
+  "share": 479,
+  "way": 480,
+  "boy": 481,
+  "smaller": 482,
+  "than": 483,
+  "look": 484,
+  "nodded": 485,
+  "weak": 486,
+  "man": 487,
+  "smooth": 488,
+  "buy": 489,
+  "already": 490,
+  "slept": 491,
+  "wake": 492,
+  "enough": 493,
+  "turned": 494,
+  "sweet": 495,
+  "gates": 496,
+  "passed": 497,
+  "their": 498,
+  "own": 499,
+  "just": 500,
+  "board": 501,
+  "its": 502,
+  "could": 503,
+  "be": 504,
+  "seen": 505,
+  "smelled": 506
+}