Training in progress, step 1

config.json

@@ -19,11 +19,11 @@
   "layer_norm_epsilon": 1e-05,
   "model_type": "gpt2",
   "n_ctx": 1024,
-  "n_embd": 128,
-  "n_head": 8,
+  "n_embd": 16,
+  "n_head": 2,
   "n_inner": null,
-  "n_layer": 8,
-  "n_positions": 40,
+  "n_layer": 2,
+  "n_positions": 30,
   "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
   "scale_attn_by_inverse_layer_idx": false,
@@ -42,5 +42,5 @@
   "torch_dtype": "float32",
   "transformers_version": "4.30.2",
   "use_cache": true,
-  "vocab_size": 88
+  "vocab_size": 74
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28101df1a2fb2780ec77258274a2f37df7dde945337050fd389001591e090773
-size 6443802
+oid sha256:e1e2aeb84675e10641cb83695751b8230c6a0d837f0543c153fd94ca800344df
+size 41986

tokenizer.json

@@ -96,88 +96,74 @@
       "<bos>": 3,
       ".": 4,
       "the": 5,
-      "and": 6,
-      ",": 7,
+      ",": 6,
+      "and": 7,
       "was": 8,
-      "it": 9,
-      "had": 10,
-      "he": 11,
-      "in": 12,
-      "Bible": 13,
-      "He": 14,
+      "he": 9,
+      "in": 10,
+      "had": 11,
+      "n<apostrophe>t": 12,
+      "it": 13,
+      "ditch": 14,
       "a": 15,
-      "to": 16,
-      "here": 17,
-      "n<apostrophe>t": 18,
-      "ditch": 19,
-      "boy": 20,
-      "<apostrophe>s": 21,
+      "Bible": 16,
+      "cows": 17,
+      "to": 18,
+      "They": 19,
+      "class": 20,
+      "But": 21,
       "by": 22,
       "knew": 23,
-      "did": 24,
-      "should": 25,
-      "help": 26,
-      "Oh": 27,
-      "they": 28,
-      "them": 29,
-      "The": 30,
-      "But": 31,
-      "there": 32,
-      "Today": 33,
-      "blackened": 34,
-      "intended": 35,
-      ":": 36,
-      "meant": 37,
-      "justice": 38,
-      "told": 39,
-      "that": 40,
-      "They": 41,
-      "happened": 42,
-      "world": 43,
-      "now": 44,
+      "of": 24,
+      ":": 25,
+      "Oh": 26,
+      "did": 27,
+      "finally": 28,
+      "mattered": 29,
+      "they": 30,
+      "He": 31,
+      "happened": 32,
+      "here": 33,
+      "there": 34,
+      "<apostrophe>s": 35,
+      "The": 36,
+      "Today": 37,
+      "Tuesday": 38,
+      "care": 39,
+      "justice": 40,
+      "now": 41,
+      "blackened": 42,
+      "boy": 43,
+      "child": 44,
       "find": 45,
-      "quite": 46,
-      "class": 47,
-      "finally": 48,
-      "Because": 49,
-      "care": 50,
-      "his": 51,
-      "leave": 52,
-      "cows": 53,
-      "finger": 54,
-      "fingers": 55,
-      "mattered": 56,
-      "Tuesday": 57,
-      "sure": 58,
-      "of": 59,
-      "none": 60,
-      "really": 61,
-      "virtue": 62,
-      "what": 63,
-      "not": 64,
-      "moved": 65,
-      "narrowed": 66,
-      "saw": 67,
-      "were": 68,
-      "then": 69,
-      "And": 70,
-      "even": 71,
-      "rising": 72,
-      "dirt": 73,
-      "child": 74,
-      "like": 75,
-      "lower": 76,
-      "lip": 77,
-      "hazel": 78,
-      "grass": 79,
-      "than": 80,
-      "darker": 81,
-      "own": 82,
-      "baby": 83,
-      "eyes": 84,
-      "quivered": 85,
-      "from": 86,
-      "silky": 87
+      "intended": 46,
+      "none": 47,
+      "Because": 48,
+      "finger": 49,
+      "fingers": 50,
+      "sure": 51,
+      "that": 52,
+      "told": 53,
+      "were": 54,
+      "world": 55,
+      "darker": 56,
+      "grass": 57,
+      "narrowed": 58,
+      "them": 59,
+      "virtue": 60,
+      "And": 61,
+      "help": 62,
+      "meant": 63,
+      "moved": 64,
+      "not": 65,
+      "really": 66,
+      "should": 67,
+      "eyes": 68,
+      "his": 69,
+      "leave": 70,
+      "quite": 71,
+      "than": 72,
+      "what": 73
     },
     "unk_token": "<unk>"
   }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54f5a7a4a96217c64d4671412656a0522cba541a91bb8741d58830b5384eb662
-size 4472
+oid sha256:e68f36608ebbacbe4b53d31e2b20b322c94ec07e610585b26ac4d0ee319c2210
+size 4408