Upload folder using huggingface_hub

shared/checkpoints/latest/config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "OllaiForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "dtype": "float32",
+  "hidden_dropout": 0.0,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "max_position_embeddings": 128,
+  "model_type": "ollai",
+  "neftune_alpha": 0.0,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 6,
+  "qk_norm": true,
+  "rezero_init": 1.0,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "transformers_version": "5.0.0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

shared/checkpoints/latest/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5379a312b06c2a442415d09adc7a80af84f3bf58dafaee6d5d67f30a90b8697
+size 800961608

shared/checkpoints/latest/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6e8a2dd90e3bf7461a6ee411256f37e25e03bba06568802098ec3c41a19ff3a
+size 203909131

shared/checkpoints/latest/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
+size 14645

shared/checkpoints/latest/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b805a545ac0239f0317adc5f9784ee2c1b4784d6e62861ead1e89f1e86d5cd5
+size 1465

shared/checkpoints/latest/trainer_state.json

@@ -0,0 +1,146 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.4840469499090759,
+  "eval_steps": 500,
+  "global_step": 305,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.003425024082200578,
+      "grad_norm": 24.18876075744629,
+      "learning_rate": 5e-05,
+      "loss": 99.9608154296875,
+      "step": 1
+    },
+    {
+      "epoch": 0.06850048164401155,
+      "grad_norm": 21.921932220458984,
+      "learning_rate": 4.97969343613113e-05,
+      "loss": 128.91767321134867,
+      "step": 20
+    },
+    {
+      "epoch": 0.1370009632880231,
+      "grad_norm": 21.391780853271484,
+      "learning_rate": 4.914814565722671e-05,
+      "loss": 107.5296630859375,
+      "step": 40
+    },
+    {
+      "epoch": 0.20550144493203468,
+      "grad_norm": 24.1998291015625,
+      "learning_rate": 4.806474728451597e-05,
+      "loss": 102.78494873046876,
+      "step": 60
+    },
+    {
+      "epoch": 0.2740019265760462,
+      "grad_norm": 22.280757904052734,
+      "learning_rate": 4.656623785870167e-05,
+      "loss": 103.28721923828125,
+      "step": 80
+    },
+    {
+      "epoch": 0.3425024082200578,
+      "grad_norm": 23.258291244506836,
+      "learning_rate": 4.4679587015226253e-05,
+      "loss": 112.2668701171875,
+      "step": 100
+    },
+    {
+      "epoch": 0.41100288986406935,
+      "grad_norm": 25.36697006225586,
+      "learning_rate": 4.24387500196217e-05,
+      "loss": 118.2769775390625,
+      "step": 120
+    },
+    {
+      "epoch": 0.4795033715080809,
+      "grad_norm": 25.141952514648438,
+      "learning_rate": 3.9884056652813184e-05,
+      "loss": 105.20911865234375,
+      "step": 140
+    },
+    {
+      "epoch": 0.2539262688047611,
+      "grad_norm": 27.3099308013916,
+      "learning_rate": 3.519045425364156e-05,
+      "loss": 85.23322211371527,
+      "step": 160
+    },
+    {
+      "epoch": 0.2856670524053563,
+      "grad_norm": 27.919233322143555,
+      "learning_rate": 3.179036545907405e-05,
+      "loss": 96.77763671875,
+      "step": 180
+    },
+    {
+      "epoch": 0.3174078360059514,
+      "grad_norm": 26.967618942260742,
+      "learning_rate": 2.8248202619522192e-05,
+      "loss": 100.84085693359376,
+      "step": 200
+    },
+    {
+      "epoch": 0.34914861960654653,
+      "grad_norm": 26.822431564331055,
+      "learning_rate": 2.4638078005620243e-05,
+      "loss": 107.72332763671875,
+      "step": 220
+    },
+    {
+      "epoch": 0.38088940320714165,
+      "grad_norm": 30.54063606262207,
+      "learning_rate": 2.1035525844485415e-05,
+      "loss": 90.114990234375,
+      "step": 240
+    },
+    {
+      "epoch": 0.41263018680773683,
+      "grad_norm": 30.880550384521484,
+      "learning_rate": 1.7515921925666052e-05,
+      "loss": 83.49360961914063,
+      "step": 260
+    },
+    {
+      "epoch": 0.44437097040833196,
+      "grad_norm": 29.54069709777832,
+      "learning_rate": 1.4152906522061048e-05,
+      "loss": 113.20718994140626,
+      "step": 280
+    },
+    {
+      "epoch": 0.4761117540089271,
+      "grad_norm": 30.88645362854004,
+      "learning_rate": 1.1016843622856923e-05,
+      "loss": 99.74405517578126,
+      "step": 300
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 434,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000000000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3950023435223040.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

shared/checkpoints/latest/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17d4250f04642c233e852ae5bcf29aa1e77409f87ce07967142291ff6119166c
+size 5201