Publish TinyGPT checkpoint

README.md

@@ -0,0 +1,66 @@
+---
+license: apache-2.0
+tags:
+  - pytorch
+  - gpt
+  - tiny-gpt
+  - causal-lm
+---
+# tiny-gpt-0.7-1m
+
+This repository contains a pretrained TinyGPT checkpoint published for public use.
+This checkpoint is provided for educational and experimentation purposes.
+
+## Artifacts
+
+- `tiny_gpt_checkpoint.pt`: training checkpoint with model and optimizer state
+- `tokenizer.model`: SentencePiece tokenizer used for training and generation
+- `config.json`: model configuration serialized from the checkpoint
+- `training_config.yaml`: training and MLflow settings used for the run
+
+## How to use
+
+Use with Transformers.
+
+Starting with `transformers >= 4.43.0`, you can run conversational inference using the `pipeline` abstraction or by leveraging the `Auto` classes with `generate()`.
+
+Make sure to update your Transformers installation via `pip install --upgrade transformers`.
+
+```python
+import torch
+import transformers
+
+model_id = "vjkhambe/tiny-gpt-0.7-1m"
+device = 0 if torch.cuda.is_available() else -1
+dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float32
+
+model = transformers.AutoModelForCausalLM.from_pretrained(
+    model_id,
+    trust_remote_code=True,
+    dtype=dtype,
+)
+tokenizer = transformers.AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
+model.generation_config.max_length = None
+model.generation_config.max_new_tokens = 64
+
+pipeline = transformers.pipeline(
+    "text-generation",
+    model=model,
+    tokenizer=tokenizer,
+    device=device,
+)
+
+print(pipeline("Hey how are you doing today?"))
+```
+
+## Training details
+
+- Base package: `tiny_gpt_pretrain`
+- Model and training configuration are stored in the checkpoint and `training_config.yaml`
+- The exported checkpoint includes optimizer state for continued fine-tuning or evaluation
+
+## License
+
+Released under the Apache-2.0 license.
+
+Target repo: `vjkhambe/tiny-gpt-0.7-1m`

config.json

@@ -0,0 +1,10 @@
+{
+  "context_length": 256,
+  "d_ff": 512,
+  "d_model": 128,
+  "dropout": 0.1,
+  "n_heads": 4,
+  "n_layers": 4,
+  "tie_embeddings": true,
+  "vocab_size": 2048
+}

export_metadata.json

@@ -0,0 +1,5 @@
+{
+  "source_checkpoint": "/mnt/diskai/workspace/2-opensource/tiny-gpt/tiny-gpt-adoption/output/versions/0.7/checkpoints/tiny_gpt_latest.pt",
+  "source_tokenizer": "/mnt/diskai/workspace/2-opensource/tiny-gpt/tiny-gpt-adoption/output/versions/0.7/data/processed/tiny_bpe.model",
+  "transformers_compatible": true
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 3,
+  "transformers_version": "5.12.1",
+  "use_cache": false
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:813362bbbb95b45f9036d411f84b4e62fedc3688817cbdb53a2c668980239c78
+size 6456888

modeling_tiny_gpt.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+from transformers import GenerationMixin, PreTrainedModel, PretrainedConfig
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+
+class TinyGPTConfig(PretrainedConfig):
+    model_type = "tiny_gpt"
+
+    def __init__(
+        self,
+        vocab_size: int = 2048,
+        context_length: int = 256,
+        n_layers: int = 4,
+        n_heads: int = 4,
+        d_model: int = 128,
+        d_ff: int = 512,
+        dropout: float = 0.1,
+        tie_embeddings: bool = True,
+        bos_token_id: int = 1,
+        eos_token_id: int = 2,
+        pad_token_id: int = 3,
+        unk_token_id: int = 0,
+        use_cache: bool = False,
+        **kwargs,
+    ) -> None:
+        is_decoder = kwargs.pop("is_decoder", True)
+        tie_word_embeddings = kwargs.pop("tie_word_embeddings", tie_embeddings)
+        use_cache = kwargs.pop("use_cache", use_cache)
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            pad_token_id=pad_token_id,
+            unk_token_id=unk_token_id,
+            use_cache=use_cache,
+            tie_word_embeddings=tie_word_embeddings,
+            is_decoder=is_decoder,
+            **kwargs,
+        )
+        self.vocab_size = vocab_size
+        self.context_length = context_length
+        self.n_layers = n_layers
+        self.n_heads = n_heads
+        self.d_model = d_model
+        self.d_ff = d_ff
+        self.dropout = dropout
+        self.tie_embeddings = tie_embeddings
+
+        # Standard Transformer aliases used by generation helpers.
+        self.hidden_size = d_model
+        self.intermediate_size = d_ff
+        self.num_attention_heads = n_heads
+        self.num_hidden_layers = n_layers
+        self.max_position_embeddings = context_length
+        self.head_dim = d_model // n_heads
+
+        if self.d_model % self.n_heads != 0:
+            raise ValueError("d_model must be divisible by n_heads")
+
+
+class TokenEmbedding(nn.Module):
+    def __init__(self, config: TinyGPTConfig) -> None:
+        super().__init__()
+        self.embedding = nn.Embedding(config.vocab_size, config.d_model)
+
+    @property
+    def weight(self) -> torch.Tensor:
+        return self.embedding.weight
+
+    def forward(self, idx: torch.Tensor) -> torch.Tensor:
+        return self.embedding(idx)
+
+
+class PositionEmbedding(nn.Module):
+    def __init__(self, config: TinyGPTConfig) -> None:
+        super().__init__()
+        self.embedding = nn.Embedding(config.context_length, config.d_model)
+
+    def forward(self, seq_len: int, device: torch.device) -> torch.Tensor:
+        positions = torch.arange(seq_len, device=device).unsqueeze(0)
+        return self.embedding(positions)
+
+
+class CausalSelfAttention(nn.Module):
+    def __init__(self, config: TinyGPTConfig) -> None:
+        super().__init__()
+        self.n_heads = config.n_heads
+        self.head_dim = config.d_model // config.n_heads
+
+        self.q_proj = nn.Linear(config.d_model, config.d_model)
+        self.k_proj = nn.Linear(config.d_model, config.d_model)
+        self.v_proj = nn.Linear(config.d_model, config.d_model)
+        self.out_proj = nn.Linear(config.d_model, config.d_model)
+        self.attn_dropout = nn.Dropout(config.dropout)
+        self.resid_dropout = nn.Dropout(config.dropout)
+
+        mask = torch.tril(torch.ones(config.context_length, config.context_length))
+        self.register_buffer(
+            "causal_mask",
+            mask.view(1, 1, config.context_length, config.context_length),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        batch_size, seq_len, d_model = x.shape
+
+        query = self._split_heads(self.q_proj(x), batch_size, seq_len)
+        key = self._split_heads(self.k_proj(x), batch_size, seq_len)
+        value = self._split_heads(self.v_proj(x), batch_size, seq_len)
+
+        scores = query @ key.transpose(-2, -1)
+        scores = scores / (self.head_dim**0.5)
+        scores = scores.masked_fill(
+            self.causal_mask[:, :, :seq_len, :seq_len] == 0,
+            float("-inf"),
+        )
+
+        attention_weights = F.softmax(scores, dim=-1)
+        attention_weights = self.attn_dropout(attention_weights)
+
+        out = attention_weights @ value
+        out = out.transpose(1, 2).contiguous().view(batch_size, seq_len, d_model)
+        out = self.out_proj(out)
+        return self.resid_dropout(out)
+
+    def _split_heads(self, x: torch.Tensor, batch_size: int, seq_len: int) -> torch.Tensor:
+        x = x.view(batch_size, seq_len, self.n_heads, self.head_dim)
+        return x.transpose(1, 2)
+
+
+class FeedForward(nn.Module):
+    def __init__(self, config: TinyGPTConfig) -> None:
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(config.d_model, config.d_ff),
+            nn.GELU(),
+            nn.Linear(config.d_ff, config.d_model),
+            nn.Dropout(config.dropout),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.net(x)
+
+
+class TransformerBlock(nn.Module):
+    def __init__(self, config: TinyGPTConfig) -> None:
+        super().__init__()
+        self.ln_1 = nn.LayerNorm(config.d_model)
+        self.attn = CausalSelfAttention(config)
+        self.ln_2 = nn.LayerNorm(config.d_model)
+        self.mlp = FeedForward(config)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = x + self.attn(self.ln_1(x))
+        x = x + self.mlp(self.ln_2(x))
+        return x
+
+
+class TinyGPTForCausalLM(PreTrainedModel, GenerationMixin):
+    config_class = TinyGPTConfig
+    base_model_prefix = "tiny_gpt"
+    main_input_name = "input_ids"
+    _tied_weights_keys = {"lm_head.weight": "token_embedding.embedding.weight"}
+
+    def __init__(self, config: TinyGPTConfig) -> None:
+        super().__init__(config)
+
+        self.token_embedding = TokenEmbedding(config)
+        self.position_embedding = PositionEmbedding(config)
+        self.dropout = nn.Dropout(config.dropout)
+        self.blocks = nn.ModuleList(TransformerBlock(config) for _ in range(config.n_layers))
+        self.final_ln = nn.LayerNorm(config.d_model)
+        self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
+
+        self.post_init()
+        self.tie_weights()
+        if getattr(self, "generation_config", None) is not None:
+            self.generation_config.use_cache = False
+            self.generation_config.cache_implementation = None
+
+    def get_input_embeddings(self) -> nn.Module:
+        return self.token_embedding.embedding
+
+    def set_input_embeddings(self, value: nn.Module) -> None:
+        self.token_embedding.embedding = value
+
+    def get_output_embeddings(self) -> nn.Module:
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings: nn.Module) -> None:
+        self.lm_head = new_embeddings
+
+    def tie_weights(self, *args, **kwargs) -> None:
+        del args, kwargs
+        if self.config.tie_embeddings:
+            self.lm_head.weight = self.token_embedding.weight
+
+    def _init_weights(self, module: nn.Module) -> None:
+        if isinstance(module, nn.Linear):
+            nn.init.normal_(module.weight, mean=0.0, std=0.02)
+            if module.bias is not None:
+                nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            nn.init.normal_(module.weight, mean=0.0, std=0.02)
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        attention_mask: torch.Tensor | None = None,
+        labels: torch.Tensor | None = None,
+        past_key_values=None,
+        use_cache: bool | None = None,
+        return_dict: bool = True,
+        **kwargs,
+    ) -> CausalLMOutputWithPast | tuple[torch.Tensor, ...]:
+        del attention_mask, past_key_values, use_cache, kwargs
+
+        batch_size, seq_len = input_ids.shape
+        if seq_len > self.config.context_length:
+            raise ValueError(
+                f"Sequence length {seq_len} exceeds context length {self.config.context_length}"
+            )
+        if labels is not None and labels.shape != input_ids.shape:
+            raise ValueError(f"labels shape {labels.shape} must match input_ids shape {input_ids.shape}")
+
+        token_embeddings = self.token_embedding(input_ids)
+        position_embeddings = self.position_embedding(seq_len, input_ids.device)
+        x = token_embeddings + position_embeddings
+        x = self.dropout(x)
+
+        for block in self.blocks:
+            x = block(x)
+
+        x = self.final_ln(x)
+        logits = self.lm_head(x)
+
+        loss = None
+        if labels is not None:
+            if seq_len < 2:
+                raise ValueError("Need at least 2 tokens to compute causal LM loss")
+            shift_logits = logits[:, :-1, :].contiguous()
+            shift_labels = labels[:, 1:].contiguous()
+            loss = F.cross_entropy(
+                shift_logits.reshape(-1, self.config.vocab_size),
+                shift_labels.reshape(-1),
+                ignore_index=-100,
+            )
+
+        if not return_dict:
+            output = (logits,)
+            if loss is not None:
+                output = (loss,) + output
+            return output
+
+        return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=None)
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids: torch.Tensor,
+        past_key_values=None,
+        attention_mask: torch.Tensor | None = None,
+        **kwargs,
+    ) -> dict[str, torch.Tensor | None]:
+        del past_key_values
+        if input_ids.shape[1] > self.config.context_length:
+            input_ids = input_ids[:, -self.config.context_length :]
+            if attention_mask is not None:
+                attention_mask = attention_mask[:, -self.config.context_length :]
+        return {
+            "input_ids": input_ids,
+            "attention_mask": attention_mask,
+            "use_cache": False,
+            **kwargs,
+        }
+
+
+TinyGPTConfig.register_for_auto_class()
+TinyGPTForCausalLM.register_for_auto_class("AutoModelForCausalLM")

tiny_bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c43518042cf12a7817c2f72fc8f14d78efb0e18b8a8ef1ec7e7502a5b7bcde28
+size 270746

tiny_bpe.vocab

@@ -0,0 +1,2048 @@
+<unk>	0
+<s>	0
+</s>	0
+<pad>	0
+▁t	-0
+he	-1
+▁a	-2
+▁s	-3
+in	-4
+▁the	-5
+▁w	-6
+ou	-7
+it	-8
+▁o	-9
+nd	-10
+er	-11
+▁“	-12
+ha	-13
+ing	-14
+re	-15
+▁b	-16
+▁to	-17
+ed	-18
+▁c	-19
+▁and	-20
+▁m	-21
+▁l	-22
+id	-23
+en	-24
+ll	-25
+▁d	-26
+▁f	-27
+ce	-28
+▁he	-29
+▁sa	-30
+on	-31
+▁of	-32
+is	-33
+▁it	-34
+as	-35
+ow	-36
+ar	-37
+▁g	-38
+le	-39
+▁A	-40
+▁n	-41
+▁she	-42
+or	-43
+▁in	-44
+ice	-45
+hat	-46
+at	-47
+▁said	-48
+▁be	-49
+▁y	-50
+▁p	-51
+se	-52
+▁ha	-53
+▁th	-54
+▁you	-55
+,”	-56
+lice	-57
+▁Alice	-58
+▁I	-59
+an	-60
+ly	-61
+ot	-62
+▁was	-63
+▁on	-64
+me	-65
+ver	-66
+st	-67
+ld	-68
+▁h	-69
+▁her	-70
+ut	-71
+gh	-72
+▁e	-73
+▁as	-74
+!”	-75
+es	-76
+out	-77
+oo	-78
+▁that	-79
+ur	-80
+ch	-81
+▁so	-82
+▁u	-83
+▁T	-84
+ent	-85
+et	-86
+▁r	-87
+ve	-88
+ght	-89
+ould	-90
+▁wit	-91
+tle	-92
+▁at	-93
+ke	-94
+ad	-95
+▁with	-96
+ter	-97
+▁re	-98
+▁_	-99
+.”	-100
+ay	-101
+▁an	-102
+▁had	-103
+ir	-104
+ck	-105
+ee	-106
+very	-107
+▁not	-108
+▁all	-109
+▁M	-110
+▁wh	-111
+her	-112
+▁k	-113
+?”	-114
+ri	-115
+▁for	-116
+ook	-117
+ea	-118
+lf	-119
+▁H	-120
+if	-121
+ain	-122
+▁do	-123
+▁The	-124
+ill	-125
+im	-126
+▁D	-127
+ittle	-128
+qu	-129
+▁up	-130
+▁kn	-131
+own	-132
+▁little	-133
+▁they	-134
+▁very	-135
+ry	-136
+▁ne	-137
+self	-138
+▁su	-139
+▁out	-140
+▁sh	-141
+ro	-142
+ell	-143
+al	-144
+ge	-145
+▁li	-146
+pp	-147
+▁this	-148
+▁but	-149
+ab	-150
+ust	-151
+ion	-152
+▁j	-153
+▁S	-154
+▁C	-155
+▁see	-156
+and	-157
+our	-158
+ouse	-159
+to	-160
+ite	-161
+▁is	-162
+▁me	-163
+▁mo	-164
+▁qu	-165
+▁look	-166
+▁ab	-167
+▁his	-168
+▁beg	-169
+▁down	-170
+ound	-171
+een	-172
+ere	-173
+hing	-174
+ight	-175
+hen	-176
+▁ag	-177
+ought	-178
+▁ca	-179
+ss	-180
+ink	-181
+▁one	-182
+▁know	-183
+▁about	-184
+▁st	-185
+▁again	-186
+ard	-187
+▁them	-188
+▁what	-189
+▁go	-190
+▁like	-191
+▁no	-192
+▁were	-193
+▁Q	-194
+▁ti	-195
+▁went	-196
+▁could	-197
+▁herself	-198
+ock	-199
+▁Qu	-200
+▁would	-201
+▁W	-202
+▁off	-203
+▁thought	-204
+▁or	-205
+▁thing	-206
+▁some	-207
+ear	-208
+▁v	-209
+▁gr	-210
+▁Queen	-211
+ong	-212
+ct	-213
+▁did	-214
+▁have	-215
+▁K	-216
+▁then	-217
+ind	-218
+ome	-219
+▁time	-220
+▁if	-221
+▁say	-222
+atter	-223
+all	-224
+ther	-225
+han	-226
+ort	-227
+ast	-228
+ers	-229
+▁into	-230
+us	-231
+che	-232
+▁get	-233
+▁when	-234
+ne	-235
+▁your	-236
+▁ex	-237
+ting	-238
+▁don	-239
+▁who	-240
+▁there	-241
+un	-242
+il	-243
+reat	-244
+urtle	-245
+▁King	-246
+▁G	-247
+▁my	-248
+nder	-249
+ul	-250
+▁*	-251
+ous	-252
+ant	-253
+▁sp	-254
+ried	-255
+▁can	-256
+▁whi	-257
+▁head	-258
+▁think	-259
+▁Turtle	-260
+li	-261
+est	-262
+king	-263
+ph	-264
+ked	-265
+▁tw	-266
+▁any	-267
+▁its	-268
+▁Mock	-269
+▁began	-270
+▁con	-271
+am	-272
+▁(	-273
+You	-274
+uch	-275
+▁by	-276
+phon	-277
+▁Gry	-278
+▁Hatter	-279
+▁Gryphon	-280
+mb	-281
+ake	-282
+▁af	-283
+ling	-284
+▁Cat	-285
+▁way	-286
+abb	-287
+▁fe	-288
+oice	-289
+▁rep	-290
+▁wor	-291
+abbit	-292
+thing	-293
+▁much	-294
+▁quite	-295
+▁R	-296
+▁ch	-297
+irst	-298
+▁other	-299
+▁their	-300
+▁voice	-301
+ig	-302
+ish	-303
+ore	-304
+ose	-305
+ure	-306
+ning	-307
+dd	-308
+▁F	-309
+▁al	-310
+▁And	-311
+▁now	-312
+ra	-313
+—”	-314
+are	-315
+▁too	-316
+▁cour	-317
+▁more	-318
+ack	-319
+ess	-320
+ough	-321
+▁him	-322
+▁how	-323
+▁Rabbit	-324
+▁B	-325
+▁po	-326
+ious	-327
+ood	-328
+ree	-329
+▁Du	-330
+▁got	-331
+▁looked	-332
+The	-333
+way	-334
+▁Wh	-335
+▁mad	-336
+▁sha	-337
+▁just	-338
+▁must	-339
+▁only	-340
+▁first	-341
+It	-342
+art	-343
+orm	-344
+▁are	-345
+od	-346
+▁L	-347
+led	-348
+▁lar	-349
+chess	-350
+▁over	-351
+▁never	-352
+▁Duchess	-353
+erp	-354
+ist	-355
+ment	-356
+▁round	-357
+ec	-358
+rom	-359
+ven	-360
+▁ar	-361
+ever	-362
+▁tur	-363
+▁Dorm	-364
+▁came	-365
+▁seem	-366
+▁such	-367
+▁tone	-368
+▁should	-369
+▁Dormouse	-370
+mp	-371
+What	-372
+fore	-373
+▁min	-374
+▁rat	-375
+▁back	-376
+▁great	-377
+▁which	-378
+ace	-379
+▁pl	-380
+▁been	-381
+▁here	-382
+▁after	-383
+ic	-384
+▁ey	-385
+▁us	-386
+▁Mar	-387
+▁rem	-388
+ep	-389
+▁‘	-390
+oot	-391
+▁from	-392
+▁hand	-393
+▁long	-394
+▁well	-395
+▁before	-396
+Oh	-397
+Wh	-398
+ag	-399
+▁P	-400
+▁gl	-401
+ence	-402
+ave	-403
+ily	-404
+leas	-405
+▁She	-406
+▁cur	-407
+▁put	-408
+▁last	-409
+▁March	-410
+▁large	-411
+au	-412
+up	-413
+xt	-414
+one	-415
+▁ke	-416
+▁le	-417
+able	-418
+ance	-419
+▁door	-420
+om	-421
+op	-422
+but	-423
+ull	-424
+▁man	-425
+▁two	-426
+▁minut	-427
+▁right	-428
+ds	-429
+ame	-430
+ass	-431
+imp	-432
+the	-433
+▁un	-434
+▁cat	-435
+onder	-436
+▁Hare	-437
+▁dear	-438
+▁moment	-439
+▁looking	-440
+end	-441
+ury	-442
+ves	-443
+ered	-444
+▁day	-445
+▁tal	-446
+▁grow	-447
+▁made	-448
+▁near	-449
+▁tell	-450
+▁heard	-451
+▁things	-452
+▁nothing	-453
+ate	-454
+ide	-455
+lied	-456
+▁bot	-457
+ation	-458
+illar	-459
+▁chan	-460
+▁eyes	-461
+▁once	-462
+▁Mouse	-463
+▁begin	-464
+▁found	-465
+erpillar	-466
+▁replied	-467
+ect	-468
+hed	-469
+ire	-470
+▁en	-471
+▁wa	-472
+ened	-473
+▁hur	-474
+▁next	-475
+▁might	-476
+▁remar	-477
+▁wonder	-478
+▁Caterpillar	-479
+lt	-480
+ned	-481
+ody	-482
+row	-483
+▁am	-484
+ving	-485
+▁add	-486
+▁every	-487
+▁going	-488
+▁three	-489
+▁under	-490
+▁seemed	-491
+th	-492
+ild	-493
+▁CH	-494
+▁cl	-495
+▁fl	-496
+▁ta	-497
+here	-498
+▁try	-499
+▁won	-500
+▁come	-501
+▁good	-502
+▁make	-503
+▁soon	-504
+▁upon	-505
+▁pleas	-506
+el	-507
+igh	-508
+ons	-509
+red	-510
+▁wr	-511
+▁tea	-512
+▁away	-513
+▁find	-514
+▁jury	-515
+▁poor	-516
+▁than	-517
+▁course	-518
+▁rather	-519
+▁without	-520
+AP	-521
+ER	-522
+APT	-523
+ise	-524
+les	-525
+ory	-526
+▁So	-527
+▁sm	-528
+That	-529
+ally	-530
+▁dre	-531
+▁tre	-532
+APTER	-533
+arden	-534
+iting	-535
+▁mean	-536
+▁garden	-537
+▁CHAPTER	-538
+No	-539
+▁E	-540
+ive	-541
+ool	-542
+oom	-543
+▁pe	-544
+▁pi	-545
+Well	-546
+body	-547
+that	-548
+other	-549
+▁does	-550
+▁felt	-551
+▁same	-552
+▁sort	-553
+▁took	-554
+▁will	-555
+▁added	-556
+Of	-557
+de	-558
+ng	-559
+ps	-560
+age	-561
+eat	-562
+get	-563
+ine	-564
+old	-565
+umb	-566
+you	-567
+▁It	-568
+▁ma	-569
+▁pa	-570
+▁pu	-571
+▁sl	-572
+ures	-573
+▁How	-574
+▁old	-575
+▁half	-576
+▁happ	-577
+▁read	-578
+▁White	-579
+▁child	-580
+▁shall	-581
+▁another	-582
+▁getting	-583
+ted	-584
+▁op	-585
+▁fan	-586
+▁wal	-587
+▁yet	-588
+right	-589
+▁call	-590
+▁wish	-591
+estion	-592
+▁There	-593
+▁words	-594
+▁minute	-595
+▁question	-596
+iz	-597
+’”	-598
+But	-599
+Why	-600
+dly	-601
+ert	-602
+ity	-603
+oke	-604
+ved	-605
+▁lo	-606
+▁tr	-607
+memb	-608
+ully	-609
+▁arm	-610
+▁eat	-611
+▁fee	-612
+▁har	-613
+▁pro	-614
+▁set	-615
+▁use	-616
+▁ever	-617
+▁face	-618
+▁sure	-619
+▁court	-620
+▁cried	-621
+▁house	-622
+▁rememb	-623
+▁curious	-624
+ah	-625
+pt	-626
+How	-627
+ful	-628
+his	-629
+oup	-630
+she	-631
+▁br	-632
+▁sn	-633
+▁“—	-634
+ecut	-635
+leep	-636
+▁bec	-637
+▁bet	-638
+▁exp	-639
+▁per	-640
+▁wat	-641
+▁even	-642
+▁feet	-643
+▁till	-644
+iously	-645
+▁being	-646
+▁tried	-647
+▁while	-648
+▁execut	-649
+▁repeat	-650
+dv	-651
+ft	-652
+And	-653
+eak	-654
+eep	-655
+kes	-656
+▁we	-657
+ause	-658
+dden	-659
+▁cro	-660
+▁end	-661
+▁fin	-662
+▁sil	-663
+ently	-664
+fully	-665
+▁hear	-666
+▁take	-667
+▁inter	-668
+▁enough	-669
+▁anything	-670
+ew	-671
+act	-672
+ble	-673
+ged	-674
+iff	-675
+oll	-676
+ten	-677
+ump	-678
+ush	-679
+▁gu	-680
+▁“_	-681
+Come	-682
+haps	-683
+iful	-684
+▁But	-685
+▁anx	-686
+▁bit	-687
+▁low	-688
+▁sat	-689
+ished	-690
+▁hast	-691
+▁less	-692
+▁part	-693
+▁side	-694
+▁supp	-695
+ertain	-696
+▁asked	-697
+▁spoke	-698
+▁table	-699
+▁sudden	-700
+▁something	-701
+.)	-702
+bst	-703
+dge	-704
+ife	-705
+▁el	-706
+▁id	-707
+▁ang	-708
+▁app	-709
+▁far	-710
+▁ran	-711
+▁sit	-712
+▁diff	-713
+▁high	-714
+▁left	-715
+▁doesn	-716
+▁turned	-717
+▁certain	-718
+▁hastily	-719
+▁talking	-720
+:—	-721
+If	-722
+cl	-723
+gs	-724
+te	-725
+aby	-726
+eal	-727
+urp	-728
+▁He	-729
+▁sw	-730
+deed	-731
+ootm	-732
+pped	-733
+▁air	-734
+▁com	-735
+▁ear	-736
+▁lea	-737
+▁pre	-738
+▁thr	-739
+▁tri	-740
+▁Bill	-741
+▁This	-742
+▁done	-743
+▁gave	-744
+▁idea	-745
+▁knew	-746
+▁list	-747
+▁pass	-748
+▁play	-749
+▁seen	-750
+▁want	-751
+▁small	-752
+▁called	-753
+▁indeed	-754
+▁remark	-755
+▁saying	-756
+▁remember	-757
+aw	-758
+bb	-759
+ho	-760
+sw	-761
+zz	-762
+▁O	-763
+_,”	-764
+ick	-765
+ile	-766
+oin	-767
+ond	-768
+oud	-769
+per	-770
+raw	-771
+ren	-772
+▁Ma	-773
+▁ac	-774
+▁sc	-775
+inah	-776
+ited	-777
+ness	-778
+outh	-779
+▁imp	-780
+▁let	-781
+▁mor	-782
+▁saw	-783
+▁sea	-784
+▁siz	-785
+ollow	-786
+▁answ	-787
+▁both	-788
+▁didn	-789
+▁puzz	-790
+▁talk	-791
+gether	-792
+ootman	-793
+▁close	-794
+▁mouse	-795
+▁mouth	-796
+▁ought	-797
+▁better	-798
+▁happen	-799
+▁itself	-800
+▁trying	-801
+▁perhaps	-802
+▁anxiously	-803
+ac	-804
+ti	-805
+Not	-806
+ach	-807
+eez	-808
+fus	-809
+lie	-810
+odo	-811
+▁ru	-812
+ears	-813
+hind	-814
+idly	-815
+lain	-816
+ople	-817
+quet	-818
+read	-819
+ssed	-820
+ways	-821
+▁adv	-822
+▁bea	-823
+▁bus	-824
+▁may	-825
+▁que	-826
+There	-827
+▁Dodo	-828
+▁Soup	-829
+▁baby	-830
+▁cats	-831
+▁cook	-832
+▁fall	-833
+▁game	-834
+▁gone	-835
+▁kept	-836
+▁room	-837
+▁shri	-838
+▁size	-839
+▁used	-840
+▁whis	-841
+sation	-842
+utiful	-843
+▁creat	-844
+▁dance	-845
+▁hands	-846
+▁speak	-847
+▁whole	-848
+▁behind	-849
+▁follow	-850
+▁people	-851
+▁please	-852
+▁However	-853
+▁hurried	-854
+▁through	-855
+▁beginning	-856
+▁certainly	-857
+io	-858
+ts	-859
+▁N	-860
+Yes	-861
+idd	-862
+not	-863
+rup	-864
+▁As	-865
+▁cr	-866
+▁ra	-867
+Then	-868
+ards	-869
+ches	-870
+esty	-871
+geon	-872
+icul	-873
+iers	-874
+raid	-875
+urpr	-876
+ying	-877
+▁Maj	-878
+▁ask	-879
+▁bir	-880
+▁glo	-881
+▁tim	-882
+irect	-883
+ither	-884
+▁deal	-885
+▁deep	-886
+▁else	-887
+▁join	-888
+▁lear	-889
+▁many	-890
+▁name	-891
+▁rest	-892
+▁shar	-893
+▁tail	-894
+erpent	-895
+ortant	-896
+▁among	-897
+▁hedge	-898
+▁hurry	-899
+▁queer	-900
+▁sound	-901
+▁still	-902
+▁surpr	-903
+▁where	-904
+▁afraid	-905
+▁change	-906
+▁confus	-907
+▁direct	-908
+▁wouldn	-909
+▁Majesty	-910
+▁suppose	-911
+▁turning	-912
+▁finished	-913
+▁interrup	-914
+▁suddenly	-915
+_,	-916
+_”	-917
+lp	-918
+ox	-919
+ue	-920
+Off	-921
+ich	-922
+ken	-923
+oes	-924
+ued	-925
+▁Pi	-926
+▁te	-927
+They	-928
+ager	-929
+ates	-930
+ched	-931
+ense	-932
+fort	-933
+ised	-934
+pper	-935
+rink	-936
+▁For	-937
+▁dis	-938
+▁dou	-939
+▁pie	-940
+▁run	-941
+▁sis	-942
+▁sto	-943
+▁ver	-944
+▁whe	-945
+ching	-946
+eared	-947
+ended	-948
+▁Will	-949
+▁cont	-950
+▁fell	-951
+▁foot	-952
+▁glad	-953
+▁grin	-954
+▁help	-955
+▁inst	-956
+▁keep	-957
+▁life	-958
+▁mind	-959
+▁reas	-960
+▁sold	-961
+▁turn	-962
+▁wasn	-963
+▁Dinah	-964
+▁eager	-965
+▁makes	-966
+▁shout	-967
+▁sleep	-968
+▁sneez	-969
+▁these	-970
+▁trial	-971
+▁Pigeon	-972
+▁always	-973
+▁conver	-974
+▁gloves	-975
+▁hardly	-976
+▁listen	-977
+▁matter	-978
+▁sister	-979
+▁waited	-980
+▁because	-981
+▁growing	-982
+▁minutes	-983
+▁silence	-984
+▁whether	-985
+▁execution	-986
+▁conversation	-987
+II	-988
+We	-989
+ie	-990
+wn	-991
+▁V	-992
+Who	-993
+air	-994
+der	-995
+for	-996
+hes	-997
+uck	-998
+▁bl	-999
+ener	-1000
+rily	-1001
+what	-1002
+▁che	-1003
+▁dec	-1004
+▁fur	-1005
+▁mar	-1006
+▁own	-1007
+▁sig	-1008
+▁why	-1009
+antly	-1010
+▁They	-1011
+▁best	-1012
+▁dist	-1013
+▁doub	-1014
+▁jump	-1015
+▁most	-1016
+▁open	-1017
+▁pool	-1018
+▁wind	-1019
+▁word	-1020
+▁belie	-1021
+▁dream	-1022
+▁gener	-1023
+▁place	-1024
+▁sharp	-1025
+▁sight	-1026
+▁tears	-1027
+▁twink	-1028
+▁bottle	-1029
+▁bright	-1030
+▁fright	-1031
+▁having	-1032
+▁nearly	-1033
+▁opened	-1034
+▁really	-1035
+▁reason	-1036
+▁walked	-1037
+▁croquet	-1038
+▁hedgeho	-1039
+▁lessons	-1040
+▁sitting	-1041
+▁witness	-1042
+▁children	-1043
+▁offended	-1044
+▁remarked	-1045
+▁repeated	-1046
+▁soldiers	-1047
+▁thinking	-1048
+▁yourself	-1049
+▁everything	-1050
+,)	-1051
+ap	-1052
+ff	-1053
+ky	-1054
+oy	-1055
+Now	-1056
+aid	-1057
+ati	-1058
+ead	-1059
+iam	-1060
+lan	-1061
+ner	-1062
+oop	-1063
+ope	-1064
+ppy	-1065
+sid	-1066
+vid	-1067
+▁An	-1068
+▁Kn	-1069
+▁Oh	-1070
+arts	-1071
+even	-1072
+room	-1073
+rowd	-1074
+temp	-1075
+tory	-1076
+▁few	-1077
+▁key	-1078
+▁opp	-1079
+▁pig	-1080
+▁sub	-1081
+▁unc	-1082
+ather	-1083
+erent	-1084
+inued	-1085
+▁Then	-1086
+▁book	-1087
+▁care	-1088
+▁fanc	-1089
+▁give	-1090
+▁gold	-1091
+▁hall	-1092
+▁hold	-1093
+▁kind	-1094
+▁pers	-1095
+▁rate	-1096
+▁simp	-1097
+▁sing	-1098
+▁wood	-1099
+▁work	-1100
+▁Knave	-1101
+▁birds	-1102
+▁crowd	-1103
+▁glass	-1104
+▁heads	-1105
+▁learn	-1106
+▁least	-1107
+▁lobst	-1108
+▁notic	-1109
+▁sever	-1110
+▁shook	-1111
+▁those	-1112
+▁tremb	-1113
+▁answer	-1114
+▁couldn	-1115
+▁though	-1116
+▁against	-1117
+▁angrily	-1118
+▁believe	-1119
+▁explain	-1120
+▁feeling	-1121
+▁puzzled	-1122
+▁serpent	-1123
+▁shouted	-1124
+▁timidly	-1125
+▁together	-1126
+▁continued	-1127
+▁creatures	-1128
+▁different	-1129
+▁interrupted	-1130
+),	-1131
+Do	-1132
+be	-1133
+ls	-1134
+mn	-1135
+po	-1136
+so	-1137
+uc	-1138
+wo	-1139
+—‘	-1140
+day	-1141
+got	-1142
+iet	-1143
+imm	-1144
+iss	-1145
+ith	-1146
+ney	-1147
+oun	-1148
+ret	-1149
+▁At	-1150
+▁co	-1151
+▁pr	-1152
+acle	-1153
+ging	-1154
+head	-1155
+orth	-1156
+owly	-1157
+time	-1158
+your	-1159
+▁You	-1160
+▁acc	-1161
+▁bat	-1162
+▁bro	-1163
+▁car	-1164
+▁cau	-1165
+▁cle	-1166
+▁del	-1167
+▁dry	-1168
+▁has	-1169
+▁our	-1170
+▁str	-1171
+▁top	-1172
+actly	-1173
+aming	-1174
+ately	-1175
+iddle	-1176
+ortun	-1177
+▁draw	-1178
+▁fact	-1179
+▁feel	-1180
+▁hard	-1181
+▁live	-1182
+▁mush	-1183
+▁nice	-1184
+▁nose	-1185
+▁rose	-1186
+▁sent	-1187
+▁doubt	-1188
+▁dread	-1189
+▁haven	-1190
+▁lying	-1191
+▁moral	-1192
+▁party	-1193
+▁piece	-1194
+▁proce	-1195
+▁ready	-1196
+▁verse	-1197
+▁watch	-1198
+▁white	-1199
+vidence	-1200
+▁asleep	-1201
+▁coming	-1202
+▁either	-1203
+▁forgot	-1204
+▁making	-1205
+▁slates	-1206
+▁slowly	-1207
+▁window	-1208
+ortunity	-1209
+▁Footman	-1210
+▁changed	-1211
+▁eagerly	-1212
+▁meaning	-1213
+▁noticed	-1214
+▁running	-1215
+▁waiting	-1216
+▁whiting	-1217
+important	-1218
+▁appeared	-1219
+▁distance	-1220
+▁followed	-1221
+▁mushroom	-1222
+▁beautiful	-1223
+▁direction	-1224
+▁gardeners	-1225
+▁opportunity	-1226
+He	-1227
+ek	-1228
+ix	-1229
+of	-1230
+wh	-1231
+—“	-1232
+.’”	-1233
+?_”	-1234
+_.”	-1235
+app	-1236
+ery	-1237
+ger	-1238
+ies	-1239
+iol	-1240
+ngl	-1241
+ssi	-1242
+▁No	-1243
+▁fr	-1244
+Just	-1245
+etch	-1246
+fter	-1247
+hesh	-1248
+ided	-1249
+ject	-1250
+lemn	-1251
+sent	-1252
+upid	-1253
+▁Bea	-1254
+▁Who	-1255
+▁Why	-1256
+▁arg	-1257
+▁bel	-1258
+▁den	-1259
+▁dro	-1260
+▁eye	-1261
+▁gir	-1262
+▁hun	-1263
+▁isn	-1264
+▁paw	-1265
+▁sec	-1266
+▁sir	-1267
+▁usu	-1268
+ardon	-1269
+aster	-1270
+chool	-1271
+imney	-1272
+iness	-1273
+ocket	-1274
+ssion	-1275
+stand	-1276
+umber	-1277
+▁Engl	-1278
+▁Five	-1279
+▁Hear	-1280
+▁Lory	-1281
+▁When	-1282
+▁chin	-1283
+▁each	-1284
+▁excl	-1285
+▁four	-1286
+▁goes	-1287
+▁hadn	-1288
+▁hair	-1289
+▁loud	-1290
+▁neck	-1291
+▁scre	-1292
+▁show	-1293
+▁shut	-1294
+▁sigh	-1295
+▁song	-1296
+▁vent	-1297
+▁viol	-1298
+eaking	-1299
+ending	-1300
+▁begun	-1301
+▁bread	-1302
+▁breat	-1303
+▁grown	-1304
+▁leave	-1305
+▁puppy	-1306
+▁quiet	-1307
+▁shoes	-1308
+▁stood	-1309
+▁story	-1310
+▁tired	-1311
+▁trees	-1312
+▁uncom	-1313
+▁world	-1314
+entures	-1315
+heshire	-1316
+onsense	-1317
+▁consid	-1318
+▁golden	-1319
+▁larger	-1320
+▁manage	-1321
+▁middle	-1322
+▁others	-1323
+▁pardon	-1324
+▁silent	-1325
+▁solemn	-1326
+atiently	-1327
+▁William	-1328
+▁chimney	-1329
+▁decided	-1330
+▁delight	-1331
+▁history	-1332
+▁pleased	-1333
+▁subject	-1334
+▁venture	-1335
+▁business	-1336
+▁difficul	-1337
+▁happened	-1338
+▁hedgehog	-1339
+▁lobsters	-1340
+▁generally	-1341
+▁surprised	-1342
+▁wondering	-1343
+▁frightened	-1344
+In	-1345
+_—	-1346
+av	-1347
+bs	-1348
+di	-1349
+mo	-1350
+ny	-1351
+sp	-1352
+ty	-1353
+▁X	-1354
+She	-1355
+aim	-1356
+ark	-1357
+ash	-1358
+bly	-1359
+box	-1360
+cho	-1361
+cup	-1362
+eek	-1363
+ied	-1364
+ign	-1365
+oof	-1366
+pig	-1367
+riz	-1368
+urn	-1369
+was	-1370
+▁In	-1371
+▁Lo	-1372
+▁On	-1373
+▁ed	-1374
+▁eg	-1375
+▁oh	-1376
+▁‘_	-1377
+Here	-1378
+abel	-1379
+aged	-1380
+cked	-1381
+ends	-1382
+ides	-1383
+imal	-1384
+inea	-1385
+ison	-1386
+less	-1387
+lite	-1388
+lves	-1389
+most	-1390
+side	-1391
+they	-1392
+ween	-1393
+▁Let	-1394
+▁Liz	-1395
+▁Soo	-1396
+▁cut	-1397
+▁fun	-1398
+▁gre	-1399
+▁hat	-1400
+▁hot	-1401
+▁inv	-1402
+▁nur	-1403
+▁sor	-1404
+Alice	-1405
+Would	-1406
+aimed	-1407
+aught	-1408
+bster	-1409
+choly	-1410
+ering	-1411
+itten	-1412
+lease	-1413
+olded	-1414
+ouble	-1415
+ssing	-1416
+tered	-1417
+wards	-1418
+▁arms	-1419
+▁beat	-1420
+▁dare	-1421
+▁exce	-1422
+▁full	-1423
+▁hour	-1424
+▁knee	-1425
+▁late	-1426
+▁love	-1427
+▁mine	-1428
+▁pair	-1429
+▁pict	-1430
+▁roof	-1431
+▁stay	-1432
+▁tast	-1433
+▁told	-1434
+butter	-1435
+litely	-1436
+ressed	-1437
+▁First	-1438
+▁along	-1439
+▁fancy	-1440
+▁grave	-1441
+▁guess	-1442
+▁liked	-1443
+▁melan	-1444
+▁stand	-1445
+▁tarts	-1446
+▁times	-1447
+▁whisp	-1448
+▁youth	-1449
+▁Hearts	-1450
+▁Lizard	-1451
+▁almost	-1452
+▁animal	-1453
+▁broken	-1454
+▁guinea	-1455
+▁inches	-1456
+▁jumped	-1457
+▁leaves	-1458
+▁marked	-1459
+▁myself	-1460
+▁pepper	-1461
+▁person	-1462
+▁rabbit	-1463
+▁school	-1464
+▁scream	-1465
+▁second	-1466
+▁shriek	-1467
+▁simple	-1468
+▁English	-1469
+▁Lobster	-1470
+▁flaming	-1471
+▁himself	-1472
+▁however	-1473
+▁present	-1474
+▁trouble	-1475
+▁writing	-1476
+▁written	-1477
+▁Cheshire	-1478
+▁evidence	-1479
+▁executed	-1480
+▁interest	-1481
+▁nonsense	-1482
+▁politely	-1483
+▁sneezing	-1484
+▁exclaimed	-1485
+▁forgotten	-1486
+▁trembling	-1487
+▁dreadfully	-1488
+▁melancholy	-1489
+▁procession	-1490
+▁understand	-1491
+▁executioner	-1492
+!’	-1493
+As	-1494
+So	-1495
+co	-1496
+gl	-1497
+we	-1498
+▁’	-1499
+Are	-1500
+Don	-1501
+Let	-1502
+anc	-1503
+ase	-1504
+aws	-1505
+eds	-1506
+erv	-1507
+gar	-1508
+ges	-1509
+how	-1510
+ibb	-1511
+iny	-1512
+ios	-1513
+lle	-1514
+men	-1515
+oor	-1516
+sit	-1517
+til	-1518
+ult	-1519
+▁(“	-1520
+▁Do	-1521
+▁ho	-1522
+▁ju	-1523
+adri	-1524
+ange	-1525
+arth	-1526
+cour	-1527
+dign	-1528
+enty	-1529
+fect	-1530
+mper	-1531
+oots	-1532
+owed	-1533
+ring	-1534
+roll	-1535
+ross	-1536
+sing	-1537
+some	-1538
+uror	-1539
+will	-1540
+▁Adv	-1541
+▁att	-1542
+▁ban	-1543
+▁box	-1544
+▁cor	-1545
+▁cry	-1546
+▁eas	-1547
+▁ent	-1548
+▁ext	-1549
+▁fig	-1550
+▁fri	-1551
+▁hon	-1552
+▁imm	-1553
+▁kid	-1554
+▁lad	-1555
+▁led	-1556
+▁mut	-1557
+▁new	-1558
+▁sky	-1559
+▁squ	-1560
+▁sul	-1561
+▁ten	-1562
+▁van	-1563
+Which	-1564
+aring	-1565
+bbing	-1566
+dered	-1567
+inger	-1568
+ookah	-1569
+rizes	-1570
+tempt	-1571
+times	-1572
+ument	-1573
+where	-1574
+would	-1575
+▁Here	-1576
+▁Just	-1577
+▁That	-1578
+▁alto	-1579
+▁case	-1580
+▁drew	-1581
+▁ears	-1582
+▁eggs	-1583
+▁fire	-1584
+▁grun	-1585
+▁home	-1586
+▁hope	-1587
+▁nerv	-1588
+▁nibb	-1589
+▁pack	-1590
+▁plan	-1591
+▁shan	-1592
+▁stop	-1593
+▁swam	-1594
+▁tree	-1595
+▁walk	-1596
+▁wand	-1597
+iosity	-1598
+▁After	-1599
+▁While	-1600
+▁aloud	-1601
+▁angry	-1602
+▁bring	-1603
+▁doing	-1604
+▁drink	-1605
+▁faces	-1606
+▁fetch	-1607
+▁juror	-1608
+▁knock	-1609
+▁means	-1610
+▁meant	-1611
+▁moved	-1612
+▁offic	-1613
+▁often	-1614
+▁order	-1615
+▁reach	-1616
+▁reply	-1617
+▁sadly	-1618
+▁seems	-1619
+▁stick	-1620
+▁throw	-1621
+▁twice	-1622
+▁usual	-1623
+▁water	-1624
+▁write	-1625
+▁wrong	-1626
+▁young	-1627
+adrille	-1628
+rawling	-1629
+▁across	-1630
+▁asking	-1631
+▁become	-1632
+▁butter	-1633
+▁fellow	-1634
+▁finger	-1635
+▁finish	-1636
+▁forget	-1637
+▁ground	-1638
+▁height	-1639
+▁hookah	-1640
+▁indign	-1641
+▁nearer	-1642
+▁nobody	-1643
+▁notice	-1644
+▁passed	-1645
+▁pocket	-1646
+▁remain	-1647
+▁shrill	-1648
+▁shrink	-1649
+▁sighed	-1650
+▁sleepy	-1651
+▁sorrow	-1652
+▁taking	-1653
+▁teacup	-1654
+▁temper	-1655
+▁waving	-1656
+fortable	-1657
+▁between	-1658
+▁crowded	-1659
+▁dropped	-1660
+▁exactly	-1661
+▁happens	-1662
+▁morning	-1663
+▁nervous	-1664
+▁quietly	-1665
+▁shouldn	-1666
+▁sounded	-1667
+▁strange	-1668
+▁treacle	-1669
+▁twinkle	-1670
+▁walking	-1671
+▁argument	-1672
+▁flamingo	-1673
+▁sentence	-1674
+▁somebody	-1675
+▁speaking	-1676
+▁surprise	-1677
+▁Quadrille	-1678
+▁confusion	-1679
+▁curiosity	-1680
+▁everybody	-1681
+▁instantly	-1682
+▁sometimes	-1683
+▁whispered	-1684
+▁altogether	-1685
+▁remembered	-1686
+▁impatiently	-1687
+▁interesting	-1688
+).	-1689
+?—	-1690
+Ah	-1691
+Be	-1692
+Ex	-1693
+On	-1694
+_.	-1695
+az	-1696
+cc	-1697
+iv	-1698
+ms	-1699
+oh	-1700
+os	-1701
+sc	-1702
+!_”	-1703
+One	-1704
+ail	-1705
+ape	-1706
+arm	-1707
+ary	-1708
+auc	-1709
+bow	-1710
+can	-1711
+duc	-1712
+edi	-1713
+ely	-1714
+enc	-1715
+hor	-1716
+itc	-1717
+itu	-1718
+las	-1719
+let	-1720
+mon	-1721
+ont	-1722
+ote	-1723
+ped	-1724
+pit	-1725
+ray	-1726
+set	-1727
+tom	-1728
+uce	-1729
+uff	-1730
+▁ob	-1731
+Call	-1732
+Hold	-1733
+Sure	-1734
+Very	-1735
+airs	-1736
+arre	-1737
+arse	-1738
+asts	-1739
+atur	-1740
+book	-1741
+dict	-1742
+elve	-1743
+enth	-1744
+fish	-1745
+glif	-1746
+hand	-1747
+have	-1748
+hole	-1749
+ider	-1750
+ince	-1751
+ings	-1752
+ivil	-1753
+lesc	-1754
+ngue	-1755
+orpo	-1756
+ount	-1757
+pigs	-1758
+this	-1759
+tree	-1760
+umbs	-1761
+vage	-1762
+well	-1763
+when	-1764
+wise	-1765
+▁All	-1766
+▁Ann	-1767
+▁Don	-1768
+▁Two	-1769
+▁VII	-1770
+▁age	-1771
+▁bar	-1772
+▁cho	-1773
+▁cup	-1774
+▁des	-1775
+▁esc	-1776
+▁fit	-1777
+▁fly	-1778
+▁hop	-1779
+▁lau	-1780
+▁lay	-1781
+▁mus	-1782
+▁occ	-1783
+▁ret	-1784
+▁sli	-1785
+▁ter	-1786
+▁wow	-1787
+▁yaw	-1788
+▁yer	-1789
+ading	-1790
+aucus	-1791
+imble	-1792
+itude	-1793
+pital	-1794
+pping	-1795
+rench	-1796
+round	-1797
+think	-1798
+▁Beau	-1799
+▁Come	-1800
+▁Down	-1801
+▁Long	-1802
+▁Said	-1803
+▁What	-1804
+▁bank	-1805
+▁bats	-1806
+▁comp	-1807
+▁dark	-1808
+▁days	-1809
+▁dead	-1810
+▁dish	-1811
+▁fast	-1812
+▁fear	-1813
+▁fill	-1814
+▁fish	-1815
+▁flow	-1816
+▁fond	-1817
+▁girl	-1818
+▁held	-1819
+▁hers	-1820
+▁hung	-1821
+▁kitc	-1822
+▁lock	-1823
+▁mice	-1824
+▁nine	-1825
+▁none	-1826
+▁pale	-1827
+▁paws	-1828
+▁peep	-1829
+▁poin	-1830
+▁real	-1831
+▁ring	-1832
+▁rule	-1833
+▁says	-1834
+▁sett	-1835
+▁smil	-1836
+▁sobs	-1837
+▁soup	-1838
+▁stra	-1839
+▁swim	-1840
+▁tiny	-1841
+▁true	-1842
+▁week	-1843
+▁wild	-1844
+Please	-1845
+atural	-1846
+ecause	-1847
+fectly	-1848
+icular	-1849
+igging	-1850
+owning	-1851
+selves	-1852
+ssible	-1853
+▁Mabel	-1854
+▁Seven	-1855
+▁alone	-1856
+▁beaut	-1857
+▁bowed	-1858
+▁clear	-1859
+▁concl	-1860
+▁cross	-1861
+▁earth	-1862
+▁elbow	-1863
+▁grass	-1864
+▁green	-1865
+▁hours	-1866
+▁judge	-1867
+▁night	-1868
+▁paper	-1869
+▁porpo	-1870
+▁quick	-1871
+▁short	-1872
+▁sides	-1873
+▁since	-1874
+▁slate	-1875
+▁start	-1876
+▁taken	-1877
+▁until	-1878
+▁upset	-1879
+▁worth	-1880
+Nothing	-1881
+ootiful	-1882
+retched	-1883
+▁Caucus	-1884
+▁Father	-1885
+▁appear	-1886
+▁arches	-1887
+▁begins	-1888
+▁behead	-1889
+▁bottom	-1890
+▁breath	-1891
+▁busily	-1892
+▁chance	-1893
+▁corner	-1894
+▁deeply	-1895
+▁encour	-1896
+▁escape	-1897
+▁except	-1898
+▁expect	-1899
+▁fallen	-1900
+▁hoarse	-1901
+▁honour	-1902
+▁immedi	-1903
+▁jurors	-1904
+▁letter	-1905
+▁likely	-1906
+▁mouths	-1907
+▁number	-1908
+▁patter	-1909
+▁quarre	-1910
+▁savage	-1911
+▁spread	-1912
+▁stupid	-1913
+▁taught	-1914
+▁telesc	-1915
+▁tongue	-1916
+▁twelve	-1917
+▁verses	-1918
+▁wanted	-1919
+▁animals	-1920
+▁capital	-1921
+▁carried	-1922
+▁grunted	-1923
+▁hearing	-1924
+▁jumping	-1925
+▁jurymen	-1926
+▁kitchen	-1927
+▁managed	-1928
+▁natural	-1929
+▁ordered	-1930
+▁passage	-1931
+▁passion	-1932
+▁players	-1933
+▁several	-1934
+▁sharply	-1935
+▁thimble	-1936
+verything	-1937
+▁answered	-1938
+▁confused	-1939
+▁contempt	-1940
+▁creature	-1941
+▁frowning	-1942
+▁officers	-1943
+▁pictures	-1944
+▁porpoise	-1945
+▁puzzling	-1946
+▁screamed	-1947
+▁severely	-1948
+▁shoulder	-1949
+▁solemnly	-1950
+▁straight	-1951
+▁vanished	-1952
+▁ventured	-1953
+▁decidedly	-1954
+▁important	-1955
+▁otherwise	-1956
+▁perfectly	-1957
+▁questions	-1958
+▁shoulders	-1959
+▁shrinking	-1960
+▁twinkling	-1961
+▁violently	-1962
+▁adventures	-1963
+▁difficulty	-1964
+▁particular	-1965
+▁suppressed	-1966
+▁indignantly	-1967
+▁thoughtfully	-1968
+▁uncomfortable	-1969
+▁	-1970
+e	-1971
+t	-1972
+a	-1973
+o	-1974
+h	-1975
+n	-1976
+i	-1977
+s	-1978
+r	-1979
+d	-1980
+l	-1981
+u	-1982
+w	-1983
+g	-1984
+,	-1985
+c	-1986
+y	-1987
+f	-1988
+m	-1989
+p	-1990
+b	-1991
+“	-1992
+”	-1993
+k	-1994
+.	-1995
+v	-1996
+’	-1997
+I	-1998
+A	-1999
+!	-2000
+_	-2001
+T	-2002
+—	-2003
+:	-2004
+H	-2005
+W	-2006
+?	-2007
+;	-2008
+M	-2009
+D	-2010
+S	-2011
+-	-2012
+x	-2013
+C	-2014
+j	-2015
+q	-2016
+R	-2017
+O	-2018
+Q	-2019
+B	-2020
+z	-2021
+K	-2022
+N	-2023
+P	-2024
+G	-2025
+Y	-2026
+E	-2027
+F	-2028
+*	-2029
+(	-2030
+)	-2031
+L	-2032
+‘	-2033
+U	-2034
+V	-2035
+J	-2036
+X	-2037
+[	-2038
+]	-2039
+0	-2040
+3	-2041
+Z	-2042
+ù	-2043

tiny_gpt_checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23089cbfa3a6249704d946c68553baa36929cd27b392421bd9f7cc19205e834a
+size 14194471

tiny_gpt_latest.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23089cbfa3a6249704d946c68553baa36929cd27b392421bd9f7cc19205e834a
+size 14194471

tokenization_tiny_gpt.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import sentencepiece as spm
+from transformers import PreTrainedTokenizer
+
+
+class TinyGPTTokenizer(PreTrainedTokenizer):
+    vocab_files_names = {"vocab_file": "tokenizer.model"}
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(
+        self,
+        vocab_file: str,
+        unk_token: str = "<unk>",
+        bos_token: str = "<s>",
+        eos_token: str = "</s>",
+        pad_token: str = "<pad>",
+        **kwargs,
+    ) -> None:
+        self.vocab_file = vocab_file
+        self.sp_model = spm.SentencePieceProcessor(model_file=vocab_file)
+        super().__init__(
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            **kwargs,
+        )
+
+    @property
+    def vocab_size(self) -> int:
+        return self.sp_model.get_piece_size()
+
+    def get_vocab(self) -> dict[str, int]:
+        vocab = {self._convert_id_to_token(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def _tokenize(self, text: str) -> list[str]:
+        return list(self.sp_model.encode(text, out_type=str))
+
+    def _convert_token_to_id(self, token: str) -> int:
+        return int(self.sp_model.piece_to_id(token))
+
+    def _convert_id_to_token(self, index: int) -> str:
+        return self.sp_model.id_to_piece(int(index))
+
+    def convert_tokens_to_string(self, tokens: list[str]) -> str:
+        return self.sp_model.decode_pieces(tokens)
+
+    def build_inputs_with_special_tokens(
+        self,
+        token_ids_0: list[int],
+        token_ids_1: list[int] | None = None,
+    ) -> list[int]:
+        if token_ids_1 is None:
+            return token_ids_0
+        return token_ids_0 + token_ids_1
+
+    def get_special_tokens_mask(
+        self,
+        token_ids_0: list[int],
+        token_ids_1: list[int] | None = None,
+        already_has_special_tokens: bool = False,
+    ) -> list[int]:
+        if already_has_special_tokens:
+            return [0] * (len(token_ids_0) + (len(token_ids_1) if token_ids_1 else 0))
+        return [0] * (len(token_ids_0) + (len(token_ids_1) if token_ids_1 else 0))
+
+    def save_vocabulary(self, save_directory: str, filename_prefix: str | None = None) -> tuple[str]:
+        if not self.vocab_file:
+            raise ValueError("No SentencePiece model file to save")
+
+        save_dir = Path(save_directory)
+        save_dir.mkdir(parents=True, exist_ok=True)
+        filename = "tokenizer.model"
+        out_path = save_dir / filename
+        shutil.copy2(self.vocab_file, out_path)
+        return (str(out_path),)
+
+
+TinyGPTTokenizer.register_for_auto_class()

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c43518042cf12a7817c2f72fc8f14d78efb0e18b8a8ef1ec7e7502a5b7bcde28
+size 270746

tokenizer_config.json

@@ -0,0 +1,49 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_tiny_gpt.TinyGPTTokenizer",
+      null
+    ]
+  },
+  "backend": "custom",
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "model_max_length": 1000000000,
+  "pad_token": "<pad>",
+  "tokenizer_class": "TinyGPTTokenizer",
+  "unk_token": "<unk>"
+}

training_config.yaml

@@ -0,0 +1,31 @@
+training:
+  input: output/versions/{VERSION}/data/raw/input.txt
+  tokenizer: output/versions/{VERSION}/data/processed/tiny_bpe.model
+  checkpoint_dir: output/versions/{VERSION}/checkpoints
+  checkpoint_prefix: tiny_gpt
+  batch_size: 64
+  max_steps: 5000
+  learning_rate: 3e-4
+  weight_decay: 0.1
+  grad_clip: 1.0
+  eval_interval: 250
+  eval_steps: 50
+  train_split: 0.9
+  seed: 1337
+  device: auto
+  generate_tokens: 160
+  keep_checkpoints: 5
+  max_tokens: null
+  show_sample: false
+  show_checkpoint: false
+
+mlflow:
+  enabled: true
+  tracking_uri: file:output/mlruns
+  experiment_name: tiny-gpt-adoption
+  run_name: null
+  log_checkpoints: true
+  log_model_config: true
+  tags:
+    repo: tiny-gpt-adoption
+    purpose: pretrain-mlflow-hf