Initial upload of fine-tuned Operating Organism 130m Mamba model

checkpoint-100/config.json

@@ -0,0 +1,42 @@
+{
+  "architectures": [
+    "MambaForCausalLM"
+  ],
+  "bos_token_id": 0,
+  "conv_kernel": 4,
+  "d_inner": 1536,
+  "d_model": 768,
+  "dtype": "bfloat16",
+  "eos_token_id": 0,
+  "expand": 2,
+  "fused_add_norm": true,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "initializer_range": 0.1,
+  "intermediate_size": 1536,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "mamba",
+  "n_layer": 24,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "pad_vocab_size_multiple": 8,
+  "rescale_prenorm_residual": false,
+  "residual_in_fp32": true,
+  "rms_norm": true,
+  "ssm_cfg": {},
+  "state_size": 16,
+  "tie_word_embeddings": true,
+  "time_step_floor": 0.0001,
+  "time_step_init_scheme": "random",
+  "time_step_max": 0.1,
+  "time_step_min": 0.001,
+  "time_step_rank": 48,
+  "time_step_scale": 1.0,
+  "transformers_version": "5.9.0",
+  "use_associative_scan": true,
+  "use_bias": false,
+  "use_cache": false,
+  "use_conv_bias": true,
+  "use_mambapy": false,
+  "vocab_size": 50280
+}

checkpoint-100/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0"
+}

checkpoint-100/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7329a69876e9f24dc04a4847deca8d90118c3e860e91f585ac72fa2417f26a6e
+size 258297008

checkpoint-100/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a8f8ac4c67befa0250c7f04125b85889995a41bc4f30edb6476dcc8f0974fe0
+size 516744314

checkpoint-100/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b66e3cc7c452b707ddac5caf0aa17618afb9bc1a0333600a22c4afb353f3165
+size 14244

checkpoint-100/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed799b255580dabe9c823e3cb2fa404cbeead7d5ca621da5f18f2bd5c558d547
+size 1064

checkpoint-100/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|padding|>",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<|endoftext|>"
+}

checkpoint-100/trainer_state.json

@@ -0,0 +1,104 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.8256880733944953,
+  "eval_steps": 500,
+  "global_step": 100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.1834862385321101,
+      "grad_norm": 10.875,
+      "learning_rate": 0.0001890909090909091,
+      "loss": 8.452513885498046,
+      "step": 10
+    },
+    {
+      "epoch": 0.3669724770642202,
+      "grad_norm": 15.6875,
+      "learning_rate": 0.00017696969696969698,
+      "loss": 7.812913513183593,
+      "step": 20
+    },
+    {
+      "epoch": 0.5504587155963303,
+      "grad_norm": 10.75,
+      "learning_rate": 0.00016484848484848487,
+      "loss": 7.122618103027344,
+      "step": 30
+    },
+    {
+      "epoch": 0.7339449541284404,
+      "grad_norm": 11.375,
+      "learning_rate": 0.00015272727272727275,
+      "loss": 6.769625854492188,
+      "step": 40
+    },
+    {
+      "epoch": 0.9174311926605505,
+      "grad_norm": 8.4375,
+      "learning_rate": 0.0001406060606060606,
+      "loss": 6.855507659912109,
+      "step": 50
+    },
+    {
+      "epoch": 1.091743119266055,
+      "grad_norm": 8.4375,
+      "learning_rate": 0.0001284848484848485,
+      "loss": 6.8293609619140625,
+      "step": 60
+    },
+    {
+      "epoch": 1.2752293577981653,
+      "grad_norm": 9.5,
+      "learning_rate": 0.00011636363636363636,
+      "loss": 5.922149658203125,
+      "step": 70
+    },
+    {
+      "epoch": 1.4587155963302751,
+      "grad_norm": 11.625,
+      "learning_rate": 0.00010424242424242425,
+      "loss": 6.009264755249023,
+      "step": 80
+    },
+    {
+      "epoch": 1.6422018348623855,
+      "grad_norm": 8.875,
+      "learning_rate": 9.212121212121214e-05,
+      "loss": 5.899383926391602,
+      "step": 90
+    },
+    {
+      "epoch": 1.8256880733944953,
+      "grad_norm": 9.375,
+      "learning_rate": 8e-05,
+      "loss": 5.953501510620117,
+      "step": 100
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 165,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 221350424739840.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-100/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c414b566b81f34719940a3164252c34233be1275d912c5cac7cb18b2f5536c4
+size 4856

checkpoint-165/config.json

@@ -0,0 +1,42 @@
+{
+  "architectures": [
+    "MambaForCausalLM"
+  ],
+  "bos_token_id": 0,
+  "conv_kernel": 4,
+  "d_inner": 1536,
+  "d_model": 768,
+  "dtype": "bfloat16",
+  "eos_token_id": 0,
+  "expand": 2,
+  "fused_add_norm": true,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "initializer_range": 0.1,
+  "intermediate_size": 1536,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "mamba",
+  "n_layer": 24,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "pad_vocab_size_multiple": 8,
+  "rescale_prenorm_residual": false,
+  "residual_in_fp32": true,
+  "rms_norm": true,
+  "ssm_cfg": {},
+  "state_size": 16,
+  "tie_word_embeddings": true,
+  "time_step_floor": 0.0001,
+  "time_step_init_scheme": "random",
+  "time_step_max": 0.1,
+  "time_step_min": 0.001,
+  "time_step_rank": 48,
+  "time_step_scale": 1.0,
+  "transformers_version": "5.9.0",
+  "use_associative_scan": true,
+  "use_bias": false,
+  "use_cache": false,
+  "use_conv_bias": true,
+  "use_mambapy": false,
+  "vocab_size": 50280
+}

checkpoint-165/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0"
+}

checkpoint-165/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07ceff0c8e6e411f27893a13ed7d2109f3a3675380066b1bf9492c8dd77d3c57
+size 258297008

checkpoint-165/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e404216b215893c940b69e27a82b8abdd1715f376fe96063426925f116b8406
+size 516744314

checkpoint-165/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cf9097d4513154245c48236b6ec5137b7ee2a21c9f58f2cba798ea275c6026f
+size 14244

checkpoint-165/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a11094029c279c22117488ea4607bb3577b330f33526e196e317d9df219157d
+size 1064

checkpoint-165/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|padding|>",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<|endoftext|>"
+}

checkpoint-165/trainer_state.json

@@ -0,0 +1,146 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 165,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.1834862385321101,
+      "grad_norm": 10.875,
+      "learning_rate": 0.0001890909090909091,
+      "loss": 8.452513885498046,
+      "step": 10
+    },
+    {
+      "epoch": 0.3669724770642202,
+      "grad_norm": 15.6875,
+      "learning_rate": 0.00017696969696969698,
+      "loss": 7.812913513183593,
+      "step": 20
+    },
+    {
+      "epoch": 0.5504587155963303,
+      "grad_norm": 10.75,
+      "learning_rate": 0.00016484848484848487,
+      "loss": 7.122618103027344,
+      "step": 30
+    },
+    {
+      "epoch": 0.7339449541284404,
+      "grad_norm": 11.375,
+      "learning_rate": 0.00015272727272727275,
+      "loss": 6.769625854492188,
+      "step": 40
+    },
+    {
+      "epoch": 0.9174311926605505,
+      "grad_norm": 8.4375,
+      "learning_rate": 0.0001406060606060606,
+      "loss": 6.855507659912109,
+      "step": 50
+    },
+    {
+      "epoch": 1.091743119266055,
+      "grad_norm": 8.4375,
+      "learning_rate": 0.0001284848484848485,
+      "loss": 6.8293609619140625,
+      "step": 60
+    },
+    {
+      "epoch": 1.2752293577981653,
+      "grad_norm": 9.5,
+      "learning_rate": 0.00011636363636363636,
+      "loss": 5.922149658203125,
+      "step": 70
+    },
+    {
+      "epoch": 1.4587155963302751,
+      "grad_norm": 11.625,
+      "learning_rate": 0.00010424242424242425,
+      "loss": 6.009264755249023,
+      "step": 80
+    },
+    {
+      "epoch": 1.6422018348623855,
+      "grad_norm": 8.875,
+      "learning_rate": 9.212121212121214e-05,
+      "loss": 5.899383926391602,
+      "step": 90
+    },
+    {
+      "epoch": 1.8256880733944953,
+      "grad_norm": 9.375,
+      "learning_rate": 8e-05,
+      "loss": 5.953501510620117,
+      "step": 100
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 4.75,
+      "learning_rate": 6.787878787878789e-05,
+      "loss": 5.726978302001953,
+      "step": 110
+    },
+    {
+      "epoch": 2.18348623853211,
+      "grad_norm": 6.875,
+      "learning_rate": 5.5757575757575766e-05,
+      "loss": 6.1194103240966795,
+      "step": 120
+    },
+    {
+      "epoch": 2.36697247706422,
+      "grad_norm": 7.28125,
+      "learning_rate": 4.3636363636363636e-05,
+      "loss": 6.045368194580078,
+      "step": 130
+    },
+    {
+      "epoch": 2.5504587155963305,
+      "grad_norm": 8.375,
+      "learning_rate": 3.151515151515151e-05,
+      "loss": 5.3934986114501955,
+      "step": 140
+    },
+    {
+      "epoch": 2.7339449541284404,
+      "grad_norm": 6.8125,
+      "learning_rate": 1.9393939393939395e-05,
+      "loss": 5.44294319152832,
+      "step": 150
+    },
+    {
+      "epoch": 2.9174311926605503,
+      "grad_norm": 6.4375,
+      "learning_rate": 7.272727272727272e-06,
+      "loss": 5.82293930053711,
+      "step": 160
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 165,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 363726577336320.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-165/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c414b566b81f34719940a3164252c34233be1275d912c5cac7cb18b2f5536c4
+size 4856

config.json

@@ -0,0 +1,42 @@
+{
+  "architectures": [
+    "MambaForCausalLM"
+  ],
+  "bos_token_id": 0,
+  "conv_kernel": 4,
+  "d_inner": 1536,
+  "d_model": 768,
+  "dtype": "bfloat16",
+  "eos_token_id": 0,
+  "expand": 2,
+  "fused_add_norm": true,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "initializer_range": 0.1,
+  "intermediate_size": 1536,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "mamba",
+  "n_layer": 24,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "pad_vocab_size_multiple": 8,
+  "rescale_prenorm_residual": false,
+  "residual_in_fp32": true,
+  "rms_norm": true,
+  "ssm_cfg": {},
+  "state_size": 16,
+  "tie_word_embeddings": true,
+  "time_step_floor": 0.0001,
+  "time_step_init_scheme": "random",
+  "time_step_max": 0.1,
+  "time_step_min": 0.001,
+  "time_step_rank": 48,
+  "time_step_scale": 1.0,
+  "transformers_version": "5.9.0",
+  "use_associative_scan": true,
+  "use_bias": false,
+  "use_cache": false,
+  "use_conv_bias": true,
+  "use_mambapy": false,
+  "vocab_size": 50280
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07ceff0c8e6e411f27893a13ed7d2109f3a3675380066b1bf9492c8dd77d3c57
+size 258297008

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|padding|>",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<|endoftext|>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c414b566b81f34719940a3164252c34233be1275d912c5cac7cb18b2f5536c4
+size 4856