Training in progress, step 100

config.json

@@ -6,16 +6,16 @@
   "attention_dropout": 0.0,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "head_dim": 128,
+  "head_dim": 256,
   "hidden_act": "silu",
-  "hidden_size": 1024,
+  "hidden_size": 2048,
   "initializer_range": 0.02,
-  "intermediate_size": 5000,
+  "intermediate_size": 8192,
   "max_position_embeddings": 2048,
   "mlp_bias": false,
   "model_type": "stockllama",
   "num_attention_heads": 8,
-  "num_hidden_layers": 8,
+  "num_hidden_layers": 12,
   "num_key_value_heads": 8,
   "pad_token_id": 0,
   "pretraining_tp": 1,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:270d4f45357a1b8e5e170ee0b59b369d295dc2b7a373c72e4e5da2cf18f2cd68
-size 994459984
+oid sha256:9b7759cd17fa076b59364644ced80fa7d1cce0dc9797030dffe2b548f4140746
+size 3958731256

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1feb4ae246ed74c133831cd522d3f268c44240db3e54cc445e171cd434558978
+oid sha256:719182d078a65dad32d29e443cd06558d8aabb2fbf857e8656103b6b9a87c3c6
 size 5176