Upload folder using huggingface_hub

snake-c73af121-checkpoint-8/config.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#define N_LAYERS 24
+#define VOCAB_SIZE 256
+#define D_MODEL 768
+#define D_INNER 1536
+#define DT_RANK 48
+#define D_STATE 16
+#define D_CONV 4

snake-c73af121-checkpoint-8/config.json

@@ -0,0 +1 @@
+{"d_model": 768, "n_layer": 24, "vocab_size": 256, "ssm_cfg": {}, "rms_norm": true, "residual_in_fp32": true, "fused_add_norm": true, "pad_vocab_size_multiple": 8, "tie_embeddings": false}

snake-c73af121-checkpoint-8/model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:133d05fdf9e28b244ddd4aecadf5b09c0dc757acaff4cce43737371f90a4a037
+size 363654144

snake-c73af121-checkpoint-8/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14c4d7148326b7de6fce61a836a7feb5e72d0d8aaa3ed15fbc266fe764d22110
+size 363746390

snake-c73af121-checkpoint-8/training_params.json

@@ -0,0 +1,18 @@
+{
+    "d_model": 768,
+    "n_layers": 24,
+    "vocab_size": 256,
+    "model_size": 90913536,
+    "learning_rate": 0.0001,
+    "seq_length": 1024,
+    "batch_size": 4,
+    "num_batches": 1241,
+    "tokenizer_path": "",
+    "dataset_path": "flpelerin/TinyStories-train-1M",
+    "model_name": "snake-c73af121",
+    "wandb_project": "snake-91M",
+    "wandb_user": "florianpelerin110304",
+    "hf_repo": "flpelerin/snake-91M",
+    "target_epoch": 12,
+    "current_epoch": 8
+}