qxsecureserver commited on Nov 14, 2024

Commit

74f10f8

verified ·

1 Parent(s): b58fe92

Upload folder using huggingface_hub

Browse files

Files changed (33) hide show

.gitattributes +2 -0
.ipynb_checkpoints/all_results-checkpoint.json +12 -0
.ipynb_checkpoints/config-checkpoint.json +35 -0
.ipynb_checkpoints/special_tokens_map-checkpoint.json +59 -0
.ipynb_checkpoints/tokenizer-checkpoint.json +3 -0
.ipynb_checkpoints/tokenizer_config-checkpoint.json +250 -0
.ipynb_checkpoints/train_results-checkpoint.json +8 -0
.ipynb_checkpoints/trainer_log-checkpoint.jsonl +155 -0
.ipynb_checkpoints/trainer_state-checkpoint.json +1123 -0
added_tokens.json +28 -0
all_results.json +12 -0
config.json +35 -0
configuration_omeganeo.py +59 -0
eval_results.json +7 -0
generation_config.json +6 -0
logs.jsonl +0 -0
merges.txt +0 -0
model-00001-of-00004.safetensors +3 -0
model-00002-of-00004.safetensors +3 -0
model-00003-of-00004.safetensors +3 -0
model-00004-of-00004.safetensors +3 -0
model.safetensors.index.json +394 -0
modeling_omeganeo.py +1343 -0
special_tokens_map.json +59 -0
tokenization_omeganeo.py +303 -0
tokenization_omeganeo_fast.py +114 -0
tokenizer.json +3 -0
tokenizer_config.json +250 -0
train_results.json +8 -0
trainer_log.jsonl +155 -0
trainer_state.json +1123 -0
training_args.bin +3 -0
vocab.json +0 -0

.gitattributes CHANGED Viewed

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+.ipynb_checkpoints/tokenizer-checkpoint.json filter=lfs diff=lfs merge=lfs -text

.ipynb_checkpoints/all_results-checkpoint.json ADDED Viewed

	@@ -0,0 +1,12 @@

+{
+    "epoch": 2.995181020635117,
+    "eval_loss": 0.37776756286621094,
+    "eval_runtime": 82.0824,
+    "eval_samples_per_second": 83.026,
+    "eval_steps_per_second": 5.19,
+    "total_flos": 504886288187392.0,
+    "train_loss": 0.33701497725527674,
+    "train_runtime": 18615.961,
+    "train_samples_per_second": 20.865,
+    "train_steps_per_second": 0.081
+}

.ipynb_checkpoints/config-checkpoint.json ADDED Viewed

	@@ -0,0 +1,35 @@

+{
+  "_name_or_path": "models/OmegaNeo-8b",
+  "architectures": [
+    "OmegaNeoForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_omeganeo.OmegaNeoConfig",
+    "AutoModel": "configuration_omeganeo.OmegaNeoModel",
+    "AutoModelForCausalLM": "modeling_omeganeo.OmegaNeoForCausalLM"
+  },
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "omeganeo",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.0",
+  "use_cache": false,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

.ipynb_checkpoints/special_tokens_map-checkpoint.json ADDED Viewed

	@@ -0,0 +1,59 @@

+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>",
+    {
+      "content": "<chain_of_thought>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</chain_of_thought>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<cite docs=",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</cite>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

.ipynb_checkpoints/tokenizer-checkpoint.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:cbfffb9fa74a617c8bf459e5b28602217e14c00715bcc2fb899ac6b591c356da
+size 11422659

.ipynb_checkpoints/tokenizer_config-checkpoint.json ADDED Viewed

	@@ -0,0 +1,250 @@

+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<chain_of_thought>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "</chain_of_thought>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<cite docs=",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "</cite>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>",
+    "<chain_of_thought>",
+    "</chain_of_thought>",
+    "<cite docs=",
+    "</cite>"
+  ],
+  "bos_token": null,
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_omeganeo.OmegaNeoTokenizer",
+      null
+    ]
+  },
+  "chat_template": "{% set pre_system_message = 'You are Omega, an AI Assistant exclusively developed, trained and powered by the scientists and engineers at QX LAB AI. QX LAB AI, based in Dubai, UAE, specializes in developing and integrating AI technologies to enhance business operations across various industries. You are built on the unique Omega architecture and trained with extensive datasets and configurations. You were developed solely by the scientists and engineers at QX LAB AI, without any external assistance from other organizations or teams.' %}{% if messages[0]['role'] == 'system' %}<|im_start|>system\n{{ pre_system_message }}\n\n{{ messages[0]['content'] }}<|im_end|>\n{% for message in messages[1:] %}{% if message['role'] == 'user' %}<|im_start|>user\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'assistant' %}<|im_start|>assistant\n{{ message['content'] }}<|im_end|>\n{% endif %}{% endfor %}{% else %}<|im_start|>system\n{{ pre_system_message }}<|im_end|>\n{% for message in messages %}{% if message['role'] == 'user' %}<|im_start|>user\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'assistant' %}<|im_start|>assistant\n{{ message['content'] }}<|im_end|>\n{% endif %}{% endfor %}{% endif %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "OmegaNeoTokenizer",
+  "unk_token": null
+}

.ipynb_checkpoints/train_results-checkpoint.json ADDED Viewed

	@@ -0,0 +1,8 @@

+{
+    "epoch": 2.995181020635117,
+    "total_flos": 504886288187392.0,
+    "train_loss": 0.33701497725527674,
+    "train_runtime": 18615.961,
+    "train_samples_per_second": 20.865,
+    "train_steps_per_second": 0.081
+}

.ipynb_checkpoints/trainer_log-checkpoint.jsonl ADDED Viewed

	@@ -0,0 +1,155 @@

+{"current_steps": 10, "total_steps": 1515, "loss": 1.4192, "learning_rate": 4.347826086956522e-06, "epoch": 0.019770171753367108, "percentage": 0.66, "elapsed_time": "0:02:04", "remaining_time": "5:12:51"}
+{"current_steps": 20, "total_steps": 1515, "loss": 0.9179, "learning_rate": 8.695652173913044e-06, "epoch": 0.039540343506734216, "percentage": 1.32, "elapsed_time": "0:03:59", "remaining_time": "4:58:39"}
+{"current_steps": 30, "total_steps": 1515, "loss": 0.6842, "learning_rate": 1.3043478260869566e-05, "epoch": 0.059310515260101324, "percentage": 1.98, "elapsed_time": "0:05:58", "remaining_time": "4:56:09"}
+{"current_steps": 40, "total_steps": 1515, "loss": 0.5843, "learning_rate": 1.739130434782609e-05, "epoch": 0.07908068701346843, "percentage": 2.64, "elapsed_time": "0:07:54", "remaining_time": "4:51:24"}
+{"current_steps": 50, "total_steps": 1515, "loss": 0.5283, "learning_rate": 1.999963411593758e-05, "epoch": 0.09885085876683554, "percentage": 3.3, "elapsed_time": "0:09:53", "remaining_time": "4:49:58"}
+{"current_steps": 60, "total_steps": 1515, "loss": 0.513, "learning_rate": 1.9995518227714435e-05, "epoch": 0.11862103052020265, "percentage": 3.96, "elapsed_time": "0:11:46", "remaining_time": "4:45:41"}
+{"current_steps": 70, "total_steps": 1515, "loss": 0.498, "learning_rate": 1.9986830984827473e-05, "epoch": 0.13839120227356974, "percentage": 4.62, "elapsed_time": "0:13:40", "remaining_time": "4:42:21"}
+{"current_steps": 80, "total_steps": 1515, "loss": 0.4801, "learning_rate": 1.9973576360304135e-05, "epoch": 0.15816137402693686, "percentage": 5.28, "elapsed_time": "0:15:35", "remaining_time": "4:39:32"}
+{"current_steps": 90, "total_steps": 1515, "loss": 0.4764, "learning_rate": 1.9955760416020185e-05, "epoch": 0.17793154578030396, "percentage": 5.94, "elapsed_time": "0:17:32", "remaining_time": "4:37:44"}
+{"current_steps": 100, "total_steps": 1515, "loss": 0.4707, "learning_rate": 1.9933391299927346e-05, "epoch": 0.19770171753367108, "percentage": 6.6, "elapsed_time": "0:19:37", "remaining_time": "4:37:38"}
+{"current_steps": 110, "total_steps": 1515, "loss": 0.4438, "learning_rate": 1.9906479242326927e-05, "epoch": 0.21747188928703817, "percentage": 7.26, "elapsed_time": "0:21:32", "remaining_time": "4:35:10"}
+{"current_steps": 120, "total_steps": 1515, "loss": 0.4507, "learning_rate": 1.987503655119108e-05, "epoch": 0.2372420610404053, "percentage": 7.92, "elapsed_time": "0:23:26", "remaining_time": "4:32:30"}
+{"current_steps": 130, "total_steps": 1515, "loss": 0.4489, "learning_rate": 1.9839077606533878e-05, "epoch": 0.2570122327937724, "percentage": 8.58, "elapsed_time": "0:25:29", "remaining_time": "4:31:33"}
+{"current_steps": 140, "total_steps": 1515, "loss": 0.4507, "learning_rate": 1.9798618853834747e-05, "epoch": 0.2767824045471395, "percentage": 9.24, "elapsed_time": "0:27:28", "remaining_time": "4:29:48"}
+{"current_steps": 150, "total_steps": 1515, "loss": 0.4542, "learning_rate": 1.975367879651728e-05, "epoch": 0.29655257630050663, "percentage": 9.9, "elapsed_time": "0:29:30", "remaining_time": "4:28:30"}
+{"current_steps": 160, "total_steps": 1515, "loss": 0.439, "learning_rate": 1.970427798748689e-05, "epoch": 0.3163227480538737, "percentage": 10.56, "elapsed_time": "0:31:23", "remaining_time": "4:25:53"}
+{"current_steps": 170, "total_steps": 1515, "loss": 0.4281, "learning_rate": 1.965043901973111e-05, "epoch": 0.3360929198072408, "percentage": 11.22, "elapsed_time": "0:33:16", "remaining_time": "4:23:17"}
+{"current_steps": 180, "total_steps": 1515, "loss": 0.4255, "learning_rate": 1.9592186515986932e-05, "epoch": 0.3558630915606079, "percentage": 11.88, "elapsed_time": "0:35:09", "remaining_time": "4:20:47"}
+{"current_steps": 190, "total_steps": 1515, "loss": 0.4275, "learning_rate": 1.9529547117479802e-05, "epoch": 0.37563326331397506, "percentage": 12.54, "elapsed_time": "0:37:05", "remaining_time": "4:18:42"}
+{"current_steps": 200, "total_steps": 1515, "loss": 0.4391, "learning_rate": 1.9462549471739546e-05, "epoch": 0.39540343506734216, "percentage": 13.2, "elapsed_time": "0:39:11", "remaining_time": "4:17:39"}
+{"current_steps": 210, "total_steps": 1515, "loss": 0.4316, "learning_rate": 1.939122421949868e-05, "epoch": 0.41517360682070925, "percentage": 13.86, "elapsed_time": "0:41:30", "remaining_time": "4:17:58"}
+{"current_steps": 220, "total_steps": 1515, "loss": 0.4309, "learning_rate": 1.931560398067915e-05, "epoch": 0.43494377857407635, "percentage": 14.52, "elapsed_time": "0:43:33", "remaining_time": "4:16:24"}
+{"current_steps": 230, "total_steps": 1515, "loss": 0.4228, "learning_rate": 1.923572333947395e-05, "epoch": 0.4547139503274435, "percentage": 15.18, "elapsed_time": "0:45:34", "remaining_time": "4:14:37"}
+{"current_steps": 240, "total_steps": 1515, "loss": 0.4132, "learning_rate": 1.9151618828530347e-05, "epoch": 0.4744841220808106, "percentage": 15.84, "elapsed_time": "0:47:29", "remaining_time": "4:12:15"}
+{"current_steps": 250, "total_steps": 1515, "loss": 0.4201, "learning_rate": 1.9063328912242047e-05, "epoch": 0.4942542938341777, "percentage": 16.5, "elapsed_time": "0:49:22", "remaining_time": "4:09:51"}
+{"current_steps": 260, "total_steps": 1515, "loss": 0.4175, "learning_rate": 1.8970893969157852e-05, "epoch": 0.5140244655875448, "percentage": 17.16, "elapsed_time": "0:51:17", "remaining_time": "4:07:35"}
+{"current_steps": 270, "total_steps": 1515, "loss": 0.4063, "learning_rate": 1.8874356273514934e-05, "epoch": 0.5337946373409119, "percentage": 17.82, "elapsed_time": "0:53:16", "remaining_time": "4:05:41"}
+{"current_steps": 280, "total_steps": 1515, "loss": 0.4218, "learning_rate": 1.8773759975905098e-05, "epoch": 0.553564809094279, "percentage": 18.48, "elapsed_time": "0:55:20", "remaining_time": "4:04:07"}
+{"current_steps": 290, "total_steps": 1515, "loss": 0.4134, "learning_rate": 1.8669151083082955e-05, "epoch": 0.5733349808476461, "percentage": 19.14, "elapsed_time": "0:57:20", "remaining_time": "4:02:13"}
+{"current_steps": 300, "total_steps": 1515, "loss": 0.3988, "learning_rate": 1.8560577436925144e-05, "epoch": 0.5931051526010133, "percentage": 19.8, "elapsed_time": "0:59:28", "remaining_time": "4:00:50"}
+{"current_steps": 310, "total_steps": 1515, "loss": 0.4175, "learning_rate": 1.8448088692550337e-05, "epoch": 0.6128753243543803, "percentage": 20.46, "elapsed_time": "1:01:28", "remaining_time": "3:58:58"}
+{"current_steps": 320, "total_steps": 1515, "loss": 0.409, "learning_rate": 1.833173629560993e-05, "epoch": 0.6326454961077475, "percentage": 21.12, "elapsed_time": "1:03:27", "remaining_time": "3:56:58"}
+{"current_steps": 330, "total_steps": 1515, "loss": 0.3997, "learning_rate": 1.8211573458759854e-05, "epoch": 0.6524156678611145, "percentage": 21.78, "elapsed_time": "1:05:25", "remaining_time": "3:54:55"}
+{"current_steps": 340, "total_steps": 1515, "loss": 0.4015, "learning_rate": 1.80876551373243e-05, "epoch": 0.6721858396144816, "percentage": 22.44, "elapsed_time": "1:07:16", "remaining_time": "3:52:30"}
+{"current_steps": 350, "total_steps": 1515, "loss": 0.4048, "learning_rate": 1.7960038004162426e-05, "epoch": 0.6919560113678488, "percentage": 23.1, "elapsed_time": "1:09:16", "remaining_time": "3:50:34"}
+{"current_steps": 360, "total_steps": 1515, "loss": 0.4075, "learning_rate": 1.7828780423749536e-05, "epoch": 0.7117261831212158, "percentage": 23.76, "elapsed_time": "1:11:17", "remaining_time": "3:48:43"}
+{"current_steps": 370, "total_steps": 1515, "loss": 0.4102, "learning_rate": 1.769394242548469e-05, "epoch": 0.731496354874583, "percentage": 24.42, "elapsed_time": "1:13:20", "remaining_time": "3:46:57"}
+{"current_steps": 380, "total_steps": 1515, "loss": 0.396, "learning_rate": 1.755558567623678e-05, "epoch": 0.7512665266279501, "percentage": 25.08, "elapsed_time": "1:15:30", "remaining_time": "3:45:33"}
+{"current_steps": 390, "total_steps": 1515, "loss": 0.4063, "learning_rate": 1.741377345214184e-05, "epoch": 0.7710366983813172, "percentage": 25.74, "elapsed_time": "1:17:33", "remaining_time": "3:43:42"}
+{"current_steps": 400, "total_steps": 1515, "loss": 0.4066, "learning_rate": 1.7268570609664253e-05, "epoch": 0.7908068701346843, "percentage": 26.4, "elapsed_time": "1:19:43", "remaining_time": "3:42:13"}
+{"current_steps": 410, "total_steps": 1515, "loss": 0.3848, "learning_rate": 1.71200435559353e-05, "epoch": 0.8105770418880514, "percentage": 27.06, "elapsed_time": "1:21:47", "remaining_time": "3:40:26"}
+{"current_steps": 420, "total_steps": 1515, "loss": 0.3882, "learning_rate": 1.6968260218382484e-05, "epoch": 0.8303472136414185, "percentage": 27.72, "elapsed_time": "1:23:45", "remaining_time": "3:38:22"}
+{"current_steps": 430, "total_steps": 1515, "loss": 0.3957, "learning_rate": 1.6813290013663573e-05, "epoch": 0.8501173853947857, "percentage": 28.38, "elapsed_time": "1:25:47", "remaining_time": "3:36:27"}
+{"current_steps": 440, "total_steps": 1515, "loss": 0.3958, "learning_rate": 1.665520381591955e-05, "epoch": 0.8698875571481527, "percentage": 29.04, "elapsed_time": "1:27:59", "remaining_time": "3:34:57"}
+{"current_steps": 450, "total_steps": 1515, "loss": 0.3846, "learning_rate": 1.6494073924361012e-05, "epoch": 0.8896577289015198, "percentage": 29.7, "elapsed_time": "1:29:50", "remaining_time": "3:32:37"}
+{"current_steps": 460, "total_steps": 1515, "loss": 0.3925, "learning_rate": 1.6329974030202803e-05, "epoch": 0.909427900654887, "percentage": 30.36, "elapsed_time": "1:31:46", "remaining_time": "3:30:28"}
+{"current_steps": 470, "total_steps": 1515, "loss": 0.3806, "learning_rate": 1.6162979182962068e-05, "epoch": 0.929198072408254, "percentage": 31.02, "elapsed_time": "1:33:50", "remaining_time": "3:28:39"}
+{"current_steps": 480, "total_steps": 1515, "loss": 0.4075, "learning_rate": 1.5993165756135035e-05, "epoch": 0.9489682441616212, "percentage": 31.68, "elapsed_time": "1:36:09", "remaining_time": "3:27:20"}
+{"current_steps": 490, "total_steps": 1515, "loss": 0.3771, "learning_rate": 1.5820611412268366e-05, "epoch": 0.9687384159149882, "percentage": 32.34, "elapsed_time": "1:38:20", "remaining_time": "3:25:42"}
+{"current_steps": 500, "total_steps": 1515, "loss": 0.3869, "learning_rate": 1.564539506744091e-05, "epoch": 0.9885085876683554, "percentage": 33.0, "elapsed_time": "1:40:15", "remaining_time": "3:23:32"}
+{"current_steps": 500, "total_steps": 1515, "eval_loss": 0.39637455344200134, "epoch": 0.9885085876683554, "percentage": 33.0, "elapsed_time": "1:41:37", "remaining_time": "3:26:17"}
+{"current_steps": 510, "total_steps": 1515, "loss": 0.356, "learning_rate": 1.546759685517219e-05, "epoch": 1.0082787594217224, "percentage": 33.66, "elapsed_time": "1:44:11", "remaining_time": "3:25:20"}
+{"current_steps": 520, "total_steps": 1515, "loss": 0.3228, "learning_rate": 1.528729808977412e-05, "epoch": 1.0280489311750896, "percentage": 34.32, "elapsed_time": "1:46:14", "remaining_time": "3:23:18"}
+{"current_steps": 530, "total_steps": 1515, "loss": 0.3182, "learning_rate": 1.5104581229162674e-05, "epoch": 1.0478191029284567, "percentage": 34.98, "elapsed_time": "1:48:23", "remaining_time": "3:21:26"}
+{"current_steps": 540, "total_steps": 1515, "loss": 0.3171, "learning_rate": 1.4919529837146529e-05, "epoch": 1.0675892746818239, "percentage": 35.64, "elapsed_time": "1:50:31", "remaining_time": "3:19:33"}
+{"current_steps": 550, "total_steps": 1515, "loss": 0.3202, "learning_rate": 1.4732228545209993e-05, "epoch": 1.087359446435191, "percentage": 36.3, "elapsed_time": "1:52:31", "remaining_time": "3:17:25"}
+{"current_steps": 560, "total_steps": 1515, "loss": 0.3121, "learning_rate": 1.4542763013807568e-05, "epoch": 1.107129618188558, "percentage": 36.96, "elapsed_time": "1:54:37", "remaining_time": "3:15:28"}
+{"current_steps": 570, "total_steps": 1515, "loss": 0.3179, "learning_rate": 1.4351219893187984e-05, "epoch": 1.126899789941925, "percentage": 37.62, "elapsed_time": "1:56:39", "remaining_time": "3:13:23"}
+{"current_steps": 580, "total_steps": 1515, "loss": 0.3112, "learning_rate": 1.4157686783765534e-05, "epoch": 1.1466699616952922, "percentage": 38.28, "elapsed_time": "1:58:36", "remaining_time": "3:11:11"}
+{"current_steps": 590, "total_steps": 1515, "loss": 0.32, "learning_rate": 1.3962252196056847e-05, "epoch": 1.1664401334486594, "percentage": 38.94, "elapsed_time": "2:00:38", "remaining_time": "3:09:08"}
+{"current_steps": 600, "total_steps": 1515, "loss": 0.3156, "learning_rate": 1.3765005510201466e-05, "epoch": 1.1862103052020265, "percentage": 39.6, "elapsed_time": "2:02:27", "remaining_time": "3:06:44"}
+{"current_steps": 610, "total_steps": 1515, "loss": 0.3196, "learning_rate": 1.3566036935084692e-05, "epoch": 1.2059804769553935, "percentage": 40.26, "elapsed_time": "2:04:31", "remaining_time": "3:04:44"}
+{"current_steps": 620, "total_steps": 1515, "loss": 0.3218, "learning_rate": 1.3365437467081406e-05, "epoch": 1.2257506487087606, "percentage": 40.92, "elapsed_time": "2:06:25", "remaining_time": "3:02:29"}
+{"current_steps": 630, "total_steps": 1515, "loss": 0.315, "learning_rate": 1.3163298848439758e-05, "epoch": 1.2455208204621278, "percentage": 41.58, "elapsed_time": "2:08:28", "remaining_time": "3:00:29"}
+{"current_steps": 640, "total_steps": 1515, "loss": 0.3221, "learning_rate": 1.2959713525323725e-05, "epoch": 1.265290992215495, "percentage": 42.24, "elapsed_time": "2:10:26", "remaining_time": "2:58:19"}
+{"current_steps": 650, "total_steps": 1515, "loss": 0.3111, "learning_rate": 1.2754774605533728e-05, "epoch": 1.285061163968862, "percentage": 42.9, "elapsed_time": "2:12:21", "remaining_time": "2:56:07"}
+{"current_steps": 660, "total_steps": 1515, "loss": 0.3151, "learning_rate": 1.2548575815924689e-05, "epoch": 1.3048313357222292, "percentage": 43.56, "elapsed_time": "2:14:22", "remaining_time": "2:54:04"}
+{"current_steps": 670, "total_steps": 1515, "loss": 0.31, "learning_rate": 1.234121145954094e-05, "epoch": 1.3246015074755961, "percentage": 44.22, "elapsed_time": "2:16:16", "remaining_time": "2:51:52"}
+{"current_steps": 680, "total_steps": 1515, "loss": 0.3139, "learning_rate": 1.2132776372487634e-05, "epoch": 1.3443716792289633, "percentage": 44.88, "elapsed_time": "2:18:13", "remaining_time": "2:49:44"}
+{"current_steps": 690, "total_steps": 1515, "loss": 0.3147, "learning_rate": 1.1923365880558345e-05, "epoch": 1.3641418509823304, "percentage": 45.54, "elapsed_time": "2:20:15", "remaining_time": "2:47:41"}
+{"current_steps": 700, "total_steps": 1515, "loss": 0.3153, "learning_rate": 1.1713075755638748e-05, "epoch": 1.3839120227356976, "percentage": 46.2, "elapsed_time": "2:22:20", "remaining_time": "2:45:43"}
+{"current_steps": 710, "total_steps": 1515, "loss": 0.3137, "learning_rate": 1.1502002171906248e-05, "epoch": 1.4036821944890647, "percentage": 46.86, "elapsed_time": "2:24:16", "remaining_time": "2:43:34"}
+{"current_steps": 720, "total_steps": 1515, "loss": 0.301, "learning_rate": 1.1290241661845649e-05, "epoch": 1.4234523662424317, "percentage": 47.52, "elapsed_time": "2:26:14", "remaining_time": "2:41:28"}
+{"current_steps": 730, "total_steps": 1515, "loss": 0.3114, "learning_rate": 1.1077891072100944e-05, "epoch": 1.4432225379957988, "percentage": 48.18, "elapsed_time": "2:28:13", "remaining_time": "2:39:23"}
+{"current_steps": 740, "total_steps": 1515, "loss": 0.3132, "learning_rate": 1.0865047519183411e-05, "epoch": 1.462992709749166, "percentage": 48.84, "elapsed_time": "2:30:12", "remaining_time": "2:37:18"}
+{"current_steps": 750, "total_steps": 1515, "loss": 0.3032, "learning_rate": 1.065180834505634e-05, "epoch": 1.482762881502533, "percentage": 49.5, "elapsed_time": "2:32:05", "remaining_time": "2:35:08"}
+{"current_steps": 760, "total_steps": 1515, "loss": 0.3113, "learning_rate": 1.0438271072616584e-05, "epoch": 1.5025330532559003, "percentage": 50.17, "elapsed_time": "2:34:24", "remaining_time": "2:33:23"}
+{"current_steps": 770, "total_steps": 1515, "loss": 0.3219, "learning_rate": 1.022453336109341e-05, "epoch": 1.5223032250092672, "percentage": 50.83, "elapsed_time": "2:36:25", "remaining_time": "2:31:21"}
+{"current_steps": 780, "total_steps": 1515, "loss": 0.3134, "learning_rate": 1.001069296138499e-05, "epoch": 1.5420733967626343, "percentage": 51.49, "elapsed_time": "2:38:17", "remaining_time": "2:29:10"}
+{"current_steps": 790, "total_steps": 1515, "loss": 0.3126, "learning_rate": 9.796847671352948e-06, "epoch": 1.5618435685160015, "percentage": 52.15, "elapsed_time": "2:40:14", "remaining_time": "2:27:03"}
+{"current_steps": 800, "total_steps": 1515, "loss": 0.3039, "learning_rate": 9.583095291095454e-06, "epoch": 1.5816137402693686, "percentage": 52.81, "elapsed_time": "2:42:10", "remaining_time": "2:24:56"}
+{"current_steps": 810, "total_steps": 1515, "loss": 0.3043, "learning_rate": 9.36953357821928e-06, "epoch": 1.6013839120227358, "percentage": 53.47, "elapsed_time": "2:44:09", "remaining_time": "2:22:52"}
+{"current_steps": 820, "total_steps": 1515, "loss": 0.296, "learning_rate": 9.156260203131299e-06, "epoch": 1.6211540837761027, "percentage": 54.13, "elapsed_time": "2:46:20", "remaining_time": "2:20:58"}
+{"current_steps": 830, "total_steps": 1515, "loss": 0.2962, "learning_rate": 8.943372704369844e-06, "epoch": 1.64092425552947, "percentage": 54.79, "elapsed_time": "2:48:20", "remaining_time": "2:18:56"}
+{"current_steps": 840, "total_steps": 1515, "loss": 0.3129, "learning_rate": 8.730968443996411e-06, "epoch": 1.660694427282837, "percentage": 55.45, "elapsed_time": "2:50:27", "remaining_time": "2:16:58"}
+{"current_steps": 850, "total_steps": 1515, "loss": 0.3119, "learning_rate": 8.519144563068037e-06, "epoch": 1.6804645990362042, "percentage": 56.11, "elapsed_time": "2:52:27", "remaining_time": "2:14:55"}
+{"current_steps": 860, "total_steps": 1515, "loss": 0.3056, "learning_rate": 8.30799793721077e-06, "epoch": 1.7002347707895713, "percentage": 56.77, "elapsed_time": "2:54:17", "remaining_time": "2:12:44"}
+{"current_steps": 870, "total_steps": 1515, "loss": 0.311, "learning_rate": 8.097625132314529e-06, "epoch": 1.7200049425429382, "percentage": 57.43, "elapsed_time": "2:56:10", "remaining_time": "2:10:36"}
+{"current_steps": 880, "total_steps": 1515, "loss": 0.3047, "learning_rate": 7.888122360369617e-06, "epoch": 1.7397751142963056, "percentage": 58.09, "elapsed_time": "2:58:07", "remaining_time": "2:08:31"}
+{"current_steps": 890, "total_steps": 1515, "loss": 0.3011, "learning_rate": 7.67958543546509e-06, "epoch": 1.7595452860496725, "percentage": 58.75, "elapsed_time": "3:00:00", "remaining_time": "2:06:24"}
+{"current_steps": 900, "total_steps": 1515, "loss": 0.2958, "learning_rate": 7.472109729969097e-06, "epoch": 1.7793154578030397, "percentage": 59.41, "elapsed_time": "3:02:00", "remaining_time": "2:04:22"}
+{"current_steps": 910, "total_steps": 1515, "loss": 0.3035, "learning_rate": 7.2657901309112325e-06, "epoch": 1.7990856295564068, "percentage": 60.07, "elapsed_time": "3:03:53", "remaining_time": "2:02:15"}
+{"current_steps": 920, "total_steps": 1515, "loss": 0.3134, "learning_rate": 7.060720996586878e-06, "epoch": 1.8188558013097738, "percentage": 60.73, "elapsed_time": "3:05:51", "remaining_time": "2:00:12"}
+{"current_steps": 930, "total_steps": 1515, "loss": 0.3055, "learning_rate": 6.85699611340333e-06, "epoch": 1.8386259730631411, "percentage": 61.39, "elapsed_time": "3:07:57", "remaining_time": "1:58:14"}
+{"current_steps": 940, "total_steps": 1515, "loss": 0.3043, "learning_rate": 6.654708652987475e-06, "epoch": 1.858396144816508, "percentage": 62.05, "elapsed_time": "3:09:46", "remaining_time": "1:56:05"}
+{"current_steps": 950, "total_steps": 1515, "loss": 0.3048, "learning_rate": 6.453951129574644e-06, "epoch": 1.8781663165698752, "percentage": 62.71, "elapsed_time": "3:11:46", "remaining_time": "1:54:03"}
+{"current_steps": 960, "total_steps": 1515, "loss": 0.3006, "learning_rate": 6.254815357698118e-06, "epoch": 1.8979364883232424, "percentage": 63.37, "elapsed_time": "3:13:50", "remaining_time": "1:52:03"}
+{"current_steps": 970, "total_steps": 1515, "loss": 0.3025, "learning_rate": 6.05739241019862e-06, "epoch": 1.9177066600766093, "percentage": 64.03, "elapsed_time": "3:15:56", "remaining_time": "1:50:05"}
+{"current_steps": 980, "total_steps": 1515, "loss": 0.2956, "learning_rate": 5.861772576573028e-06, "epoch": 1.9374768318299767, "percentage": 64.69, "elapsed_time": "3:18:00", "remaining_time": "1:48:06"}
+{"current_steps": 990, "total_steps": 1515, "loss": 0.3091, "learning_rate": 5.668045321681328e-06, "epoch": 1.9572470035833436, "percentage": 65.35, "elapsed_time": "3:20:05", "remaining_time": "1:46:06"}
+{"current_steps": 1000, "total_steps": 1515, "loss": 0.2999, "learning_rate": 5.476299244830734e-06, "epoch": 1.9770171753367107, "percentage": 66.01, "elapsed_time": "3:22:15", "remaining_time": "1:44:09"}
+{"current_steps": 1000, "total_steps": 1515, "eval_loss": 0.3708229959011078, "epoch": 1.9770171753367107, "percentage": 66.01, "elapsed_time": "3:23:37", "remaining_time": "1:44:51"}
+{"current_steps": 1010, "total_steps": 1515, "loss": 0.3066, "learning_rate": 5.28662203925562e-06, "epoch": 1.9967873470900779, "percentage": 66.67, "elapsed_time": "3:26:12", "remaining_time": "1:43:06"}
+{"current_steps": 1020, "total_steps": 1515, "loss": 0.2574, "learning_rate": 5.099100452011876e-06, "epoch": 2.016557518843445, "percentage": 67.33, "elapsed_time": "3:28:10", "remaining_time": "1:41:01"}
+{"current_steps": 1030, "total_steps": 1515, "loss": 0.2297, "learning_rate": 4.9138202443039625e-06, "epoch": 2.036327690596812, "percentage": 67.99, "elapsed_time": "3:30:18", "remaining_time": "1:39:01"}
+{"current_steps": 1040, "total_steps": 1515, "loss": 0.2445, "learning_rate": 4.730866152262841e-06, "epoch": 2.056097862350179, "percentage": 68.65, "elapsed_time": "3:32:14", "remaining_time": "1:36:56"}
+{"current_steps": 1050, "total_steps": 1515, "loss": 0.2505, "learning_rate": 4.550321848192731e-06, "epoch": 2.0758680341035465, "percentage": 69.31, "elapsed_time": "3:34:12", "remaining_time": "1:34:51"}
+{"current_steps": 1060, "total_steps": 1515, "loss": 0.2494, "learning_rate": 4.3722699023043634e-06, "epoch": 2.0956382058569134, "percentage": 69.97, "elapsed_time": "3:36:10", "remaining_time": "1:32:47"}
+{"current_steps": 1070, "total_steps": 1515, "loss": 0.239, "learning_rate": 4.196791744952291e-06, "epoch": 2.1154083776102803, "percentage": 70.63, "elapsed_time": "3:38:09", "remaining_time": "1:30:43"}
+{"current_steps": 1080, "total_steps": 1515, "loss": 0.2463, "learning_rate": 4.023967629393498e-06, "epoch": 2.1351785493636477, "percentage": 71.29, "elapsed_time": "3:40:19", "remaining_time": "1:28:44"}
+{"current_steps": 1090, "total_steps": 1515, "loss": 0.2455, "learning_rate": 3.853876595084332e-06, "epoch": 2.1549487211170146, "percentage": 71.95, "elapsed_time": "3:42:18", "remaining_time": "1:26:40"}
+{"current_steps": 1100, "total_steps": 1515, "loss": 0.2406, "learning_rate": 3.68659643153258e-06, "epoch": 2.174718892870382, "percentage": 72.61, "elapsed_time": "3:44:21", "remaining_time": "1:24:38"}
+{"current_steps": 1110, "total_steps": 1515, "loss": 0.2333, "learning_rate": 3.5222036427211704e-06, "epoch": 2.194489064623749, "percentage": 73.27, "elapsed_time": "3:46:30", "remaining_time": "1:22:38"}
+{"current_steps": 1120, "total_steps": 1515, "loss": 0.2397, "learning_rate": 3.360773412119801e-06, "epoch": 2.214259236377116, "percentage": 73.93, "elapsed_time": "3:48:26", "remaining_time": "1:20:34"}
+{"current_steps": 1130, "total_steps": 1515, "loss": 0.2394, "learning_rate": 3.202379568300511e-06, "epoch": 2.2340294081304832, "percentage": 74.59, "elapsed_time": "3:50:22", "remaining_time": "1:18:29"}
+{"current_steps": 1140, "total_steps": 1515, "loss": 0.2319, "learning_rate": 3.0470945511728856e-06, "epoch": 2.25379957988385, "percentage": 75.25, "elapsed_time": "3:52:14", "remaining_time": "1:16:23"}
+{"current_steps": 1150, "total_steps": 1515, "loss": 0.2461, "learning_rate": 2.8949893788543505e-06, "epoch": 2.2735697516372175, "percentage": 75.91, "elapsed_time": "3:54:16", "remaining_time": "1:14:21"}
+{"current_steps": 1160, "total_steps": 1515, "loss": 0.2391, "learning_rate": 2.74613361519073e-06, "epoch": 2.2933399233905845, "percentage": 76.57, "elapsed_time": "3:56:14", "remaining_time": "1:12:17"}
+{"current_steps": 1170, "total_steps": 1515, "loss": 0.2532, "learning_rate": 2.60059533794187e-06, "epoch": 2.3131100951439514, "percentage": 77.23, "elapsed_time": "3:58:16", "remaining_time": "1:10:15"}
+{"current_steps": 1180, "total_steps": 1515, "loss": 0.2442, "learning_rate": 2.458441107646935e-06, "epoch": 2.3328802668973188, "percentage": 77.89, "elapsed_time": "4:00:18", "remaining_time": "1:08:13"}
+{"current_steps": 1190, "total_steps": 1515, "loss": 0.2374, "learning_rate": 2.3197359371835802e-06, "epoch": 2.3526504386506857, "percentage": 78.55, "elapsed_time": "4:02:20", "remaining_time": "1:06:11"}
+{"current_steps": 1200, "total_steps": 1515, "loss": 0.2383, "learning_rate": 2.1845432620349338e-06, "epoch": 2.372420610404053, "percentage": 79.21, "elapsed_time": "4:04:26", "remaining_time": "1:04:09"}
+{"current_steps": 1210, "total_steps": 1515, "loss": 0.2388, "learning_rate": 2.0529249112779824e-06, "epoch": 2.39219078215742, "percentage": 79.87, "elapsed_time": "4:06:28", "remaining_time": "1:02:07"}
+{"current_steps": 1220, "total_steps": 1515, "loss": 0.2437, "learning_rate": 1.924941079306647e-06, "epoch": 2.411960953910787, "percentage": 80.53, "elapsed_time": "4:08:32", "remaining_time": "1:00:05"}
+{"current_steps": 1230, "total_steps": 1515, "loss": 0.2482, "learning_rate": 1.8006502983024332e-06, "epoch": 2.4317311256641543, "percentage": 81.19, "elapsed_time": "4:10:31", "remaining_time": "0:58:02"}
+{"current_steps": 1240, "total_steps": 1515, "loss": 0.2313, "learning_rate": 1.6801094114653082e-06, "epoch": 2.451501297417521, "percentage": 81.85, "elapsed_time": "4:12:30", "remaining_time": "0:56:00"}
+{"current_steps": 1250, "total_steps": 1515, "loss": 0.2403, "learning_rate": 1.5633735470170108e-06, "epoch": 2.4712714691708886, "percentage": 82.51, "elapsed_time": "4:14:44", "remaining_time": "0:54:00"}
+{"current_steps": 1260, "total_steps": 1515, "loss": 0.2442, "learning_rate": 1.4504960929886746e-06, "epoch": 2.4910416409242555, "percentage": 83.17, "elapsed_time": "4:16:46", "remaining_time": "0:51:57"}
+{"current_steps": 1270, "total_steps": 1515, "loss": 0.239, "learning_rate": 1.3415286728043374e-06, "epoch": 2.5108118126776224, "percentage": 83.83, "elapsed_time": "4:18:39", "remaining_time": "0:49:54"}
+{"current_steps": 1280, "total_steps": 1515, "loss": 0.2385, "learning_rate": 1.2365211216714523e-06, "epoch": 2.53058198443099, "percentage": 84.49, "elapsed_time": "4:20:41", "remaining_time": "0:47:51"}
+{"current_steps": 1290, "total_steps": 1515, "loss": 0.2482, "learning_rate": 1.135521463789232e-06, "epoch": 2.5503521561843567, "percentage": 85.15, "elapsed_time": "4:22:35", "remaining_time": "0:45:48"}
+{"current_steps": 1300, "total_steps": 1515, "loss": 0.2381, "learning_rate": 1.0385758903852406e-06, "epoch": 2.570122327937724, "percentage": 85.81, "elapsed_time": "4:24:36", "remaining_time": "0:43:45"}
+{"current_steps": 1310, "total_steps": 1515, "loss": 0.2405, "learning_rate": 9.457287385902836e-07, "epoch": 2.589892499691091, "percentage": 86.47, "elapsed_time": "4:26:34", "remaining_time": "0:41:43"}
+{"current_steps": 1320, "total_steps": 1515, "loss": 0.2304, "learning_rate": 8.570224711612385e-07, "epoch": 2.6096626714444584, "percentage": 87.13, "elapsed_time": "4:28:30", "remaining_time": "0:39:39"}
+{"current_steps": 1330, "total_steps": 1515, "loss": 0.2259, "learning_rate": 7.724976570611254e-07, "epoch": 2.6294328431978253, "percentage": 87.79, "elapsed_time": "4:30:25", "remaining_time": "0:37:36"}
+{"current_steps": 1340, "total_steps": 1515, "loss": 0.2318, "learning_rate": 6.921929529052729e-07, "epoch": 2.6492030149511923, "percentage": 88.45, "elapsed_time": "4:32:27", "remaining_time": "0:35:34"}
+{"current_steps": 1350, "total_steps": 1515, "loss": 0.2377, "learning_rate": 6.161450852820872e-07, "epoch": 2.6689731867045596, "percentage": 89.11, "elapsed_time": "4:34:24", "remaining_time": "0:33:32"}
+{"current_steps": 1360, "total_steps": 1515, "loss": 0.2402, "learning_rate": 5.443888339565018e-07, "epoch": 2.6887433584579266, "percentage": 89.77, "elapsed_time": "4:36:35", "remaining_time": "0:31:31"}
+{"current_steps": 1370, "total_steps": 1515, "loss": 0.2401, "learning_rate": 4.769570159637782e-07, "epoch": 2.7085135302112935, "percentage": 90.43, "elapsed_time": "4:38:28", "remaining_time": "0:29:28"}
+{"current_steps": 1380, "total_steps": 1515, "loss": 0.2306, "learning_rate": 4.138804706009547e-07, "epoch": 2.728283701964661, "percentage": 91.09, "elapsed_time": "4:40:32", "remaining_time": "0:27:26"}
+{"current_steps": 1390, "total_steps": 1515, "loss": 0.2407, "learning_rate": 3.5518804532278736e-07, "epoch": 2.748053873718028, "percentage": 91.75, "elapsed_time": "4:42:22", "remaining_time": "0:25:23"}
+{"current_steps": 1400, "total_steps": 1515, "loss": 0.227, "learning_rate": 3.0090658254864013e-07, "epoch": 2.767824045471395, "percentage": 92.41, "elapsed_time": "4:44:24", "remaining_time": "0:23:21"}
+{"current_steps": 1410, "total_steps": 1515, "loss": 0.2306, "learning_rate": 2.5106090738636014e-07, "epoch": 2.787594217224762, "percentage": 93.07, "elapsed_time": "4:46:27", "remaining_time": "0:21:19"}
+{"current_steps": 1420, "total_steps": 1515, "loss": 0.242, "learning_rate": 2.0567381627875639e-07, "epoch": 2.8073643889781295, "percentage": 93.73, "elapsed_time": "4:48:30", "remaining_time": "0:19:18"}
+{"current_steps": 1430, "total_steps": 1515, "loss": 0.2309, "learning_rate": 1.6476606657786386e-07, "epoch": 2.8271345607314964, "percentage": 94.39, "elapsed_time": "4:50:28", "remaining_time": "0:17:15"}
+{"current_steps": 1440, "total_steps": 1515, "loss": 0.2287, "learning_rate": 1.2835636705176403e-07, "epoch": 2.8469047324848633, "percentage": 95.05, "elapsed_time": "4:52:21", "remaining_time": "0:15:13"}
+{"current_steps": 1450, "total_steps": 1515, "loss": 0.2469, "learning_rate": 9.64613693283123e-08, "epoch": 2.8666749042382307, "percentage": 95.71, "elapsed_time": "4:54:24", "remaining_time": "0:13:11"}
+{"current_steps": 1460, "total_steps": 1515, "loss": 0.2458, "learning_rate": 6.909566027966996e-08, "epoch": 2.8864450759915976, "percentage": 96.37, "elapsed_time": "4:56:19", "remaining_time": "0:11:09"}
+{"current_steps": 1470, "total_steps": 1515, "loss": 0.2456, "learning_rate": 4.6271755351144923e-08, "epoch": 2.9062152477449645, "percentage": 97.03, "elapsed_time": "4:58:22", "remaining_time": "0:09:08"}
+{"current_steps": 1480, "total_steps": 1515, "loss": 0.2413, "learning_rate": 2.8000092837368042e-08, "epoch": 2.925985419498332, "percentage": 97.69, "elapsed_time": "5:00:13", "remaining_time": "0:07:06"}
+{"current_steps": 1490, "total_steps": 1515, "loss": 0.2388, "learning_rate": 1.4289029108436325e-08, "epoch": 2.945755591251699, "percentage": 98.35, "elapsed_time": "5:02:13", "remaining_time": "0:05:04"}
+{"current_steps": 1500, "total_steps": 1515, "loss": 0.2454, "learning_rate": 5.14483478821437e-09, "epoch": 2.965525763005066, "percentage": 99.01, "elapsed_time": "5:04:19", "remaining_time": "0:03:02"}
+{"current_steps": 1500, "total_steps": 1515, "eval_loss": 0.37777501344680786, "epoch": 2.965525763005066, "percentage": 99.01, "elapsed_time": "5:05:41", "remaining_time": "0:03:03"}
+{"current_steps": 1510, "total_steps": 1515, "loss": 0.2349, "learning_rate": 5.716918865195098e-10, "epoch": 2.985295934758433, "percentage": 99.67, "elapsed_time": "5:08:31", "remaining_time": "0:01:01"}
+{"current_steps": 1515, "total_steps": 1515, "epoch": 2.995181020635117, "percentage": 100.0, "elapsed_time": "5:10:14", "remaining_time": "0:00:00"}

.ipynb_checkpoints/trainer_state-checkpoint.json ADDED Viewed

	@@ -0,0 +1,1123 @@

+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.995181020635117,
+  "eval_steps": 500,
+  "global_step": 1515,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.019770171753367108,
+      "grad_norm": 8.273716970639242,
+      "learning_rate": 4.347826086956522e-06,
+      "loss": 1.4192,
+      "step": 10
+    },
+    {
+      "epoch": 0.039540343506734216,
+      "grad_norm": 2.348437423787933,
+      "learning_rate": 8.695652173913044e-06,
+      "loss": 0.9179,
+      "step": 20
+    },
+    {
+      "epoch": 0.059310515260101324,
+      "grad_norm": 2.0183151255256266,
+      "learning_rate": 1.3043478260869566e-05,
+      "loss": 0.6842,
+      "step": 30
+    },
+    {
+      "epoch": 0.07908068701346843,
+      "grad_norm": 1.7096945603933598,
+      "learning_rate": 1.739130434782609e-05,
+      "loss": 0.5843,
+      "step": 40
+    },
+    {
+      "epoch": 0.09885085876683554,
+      "grad_norm": 1.2099539800951016,
+      "learning_rate": 1.999963411593758e-05,
+      "loss": 0.5283,
+      "step": 50
+    },
+    {
+      "epoch": 0.11862103052020265,
+      "grad_norm": 1.539237571955243,
+      "learning_rate": 1.9995518227714435e-05,
+      "loss": 0.513,
+      "step": 60
+    },
+    {
+      "epoch": 0.13839120227356974,
+      "grad_norm": 0.9407018526102695,
+      "learning_rate": 1.9986830984827473e-05,
+      "loss": 0.498,
+      "step": 70
+    },
+    {
+      "epoch": 0.15816137402693686,
+      "grad_norm": 0.8754091153209901,
+      "learning_rate": 1.9973576360304135e-05,
+      "loss": 0.4801,
+      "step": 80
+    },
+    {
+      "epoch": 0.17793154578030396,
+      "grad_norm": 0.8624602134273651,
+      "learning_rate": 1.9955760416020185e-05,
+      "loss": 0.4764,
+      "step": 90
+    },
+    {
+      "epoch": 0.19770171753367108,
+      "grad_norm": 0.8628999100665943,
+      "learning_rate": 1.9933391299927346e-05,
+      "loss": 0.4707,
+      "step": 100
+    },
+    {
+      "epoch": 0.21747188928703817,
+      "grad_norm": 0.8080598017374558,
+      "learning_rate": 1.9906479242326927e-05,
+      "loss": 0.4438,
+      "step": 110
+    },
+    {
+      "epoch": 0.2372420610404053,
+      "grad_norm": 0.7079645225164415,
+      "learning_rate": 1.987503655119108e-05,
+      "loss": 0.4507,
+      "step": 120
+    },
+    {
+      "epoch": 0.2570122327937724,
+      "grad_norm": 0.8311840484882397,
+      "learning_rate": 1.9839077606533878e-05,
+      "loss": 0.4489,
+      "step": 130
+    },
+    {
+      "epoch": 0.2767824045471395,
+      "grad_norm": 0.8147624642320691,
+      "learning_rate": 1.9798618853834747e-05,
+      "loss": 0.4507,
+      "step": 140
+    },
+    {
+      "epoch": 0.29655257630050663,
+      "grad_norm": 0.8371409045137015,
+      "learning_rate": 1.975367879651728e-05,
+      "loss": 0.4542,
+      "step": 150
+    },
+    {
+      "epoch": 0.3163227480538737,
+      "grad_norm": 0.7237155579293956,
+      "learning_rate": 1.970427798748689e-05,
+      "loss": 0.439,
+      "step": 160
+    },
+    {
+      "epoch": 0.3360929198072408,
+      "grad_norm": 0.6804376236541478,
+      "learning_rate": 1.965043901973111e-05,
+      "loss": 0.4281,
+      "step": 170
+    },
+    {
+      "epoch": 0.3558630915606079,
+      "grad_norm": 0.7087718772894039,
+      "learning_rate": 1.9592186515986932e-05,
+      "loss": 0.4255,
+      "step": 180
+    },
+    {
+      "epoch": 0.37563326331397506,
+      "grad_norm": 0.6404671722237,
+      "learning_rate": 1.9529547117479802e-05,
+      "loss": 0.4275,
+      "step": 190
+    },
+    {
+      "epoch": 0.39540343506734216,
+      "grad_norm": 0.6613559685528836,
+      "learning_rate": 1.9462549471739546e-05,
+      "loss": 0.4391,
+      "step": 200
+    },
+    {
+      "epoch": 0.41517360682070925,
+      "grad_norm": 0.7702640494158857,
+      "learning_rate": 1.939122421949868e-05,
+      "loss": 0.4316,
+      "step": 210
+    },
+    {
+      "epoch": 0.43494377857407635,
+      "grad_norm": 0.6687391965346136,
+      "learning_rate": 1.931560398067915e-05,
+      "loss": 0.4309,
+      "step": 220
+    },
+    {
+      "epoch": 0.4547139503274435,
+      "grad_norm": 0.6455592214289195,
+      "learning_rate": 1.923572333947395e-05,
+      "loss": 0.4228,
+      "step": 230
+    },
+    {
+      "epoch": 0.4744841220808106,
+      "grad_norm": 0.6638256696831698,
+      "learning_rate": 1.9151618828530347e-05,
+      "loss": 0.4132,
+      "step": 240
+    },
+    {
+      "epoch": 0.4942542938341777,
+      "grad_norm": 0.6035529106490282,
+      "learning_rate": 1.9063328912242047e-05,
+      "loss": 0.4201,
+      "step": 250
+    },
+    {
+      "epoch": 0.5140244655875448,
+      "grad_norm": 0.6181775363815968,
+      "learning_rate": 1.8970893969157852e-05,
+      "loss": 0.4175,
+      "step": 260
+    },
+    {
+      "epoch": 0.5337946373409119,
+      "grad_norm": 0.6133100135259807,
+      "learning_rate": 1.8874356273514934e-05,
+      "loss": 0.4063,
+      "step": 270
+    },
+    {
+      "epoch": 0.553564809094279,
+      "grad_norm": 0.5966662360815947,
+      "learning_rate": 1.8773759975905098e-05,
+      "loss": 0.4218,
+      "step": 280
+    },
+    {
+      "epoch": 0.5733349808476461,
+      "grad_norm": 0.6703060027446681,
+      "learning_rate": 1.8669151083082955e-05,
+      "loss": 0.4134,
+      "step": 290
+    },
+    {
+      "epoch": 0.5931051526010133,
+      "grad_norm": 0.6252415388725653,
+      "learning_rate": 1.8560577436925144e-05,
+      "loss": 0.3988,
+      "step": 300
+    },
+    {
+      "epoch": 0.6128753243543803,
+      "grad_norm": 0.6060817440235046,
+      "learning_rate": 1.8448088692550337e-05,
+      "loss": 0.4175,
+      "step": 310
+    },
+    {
+      "epoch": 0.6326454961077475,
+      "grad_norm": 0.6593209743601871,
+      "learning_rate": 1.833173629560993e-05,
+      "loss": 0.409,
+      "step": 320
+    },
+    {
+      "epoch": 0.6524156678611145,
+      "grad_norm": 0.5982036518358108,
+      "learning_rate": 1.8211573458759854e-05,
+      "loss": 0.3997,
+      "step": 330
+    },
+    {
+      "epoch": 0.6721858396144816,
+      "grad_norm": 0.5833507470903769,
+      "learning_rate": 1.80876551373243e-05,
+      "loss": 0.4015,
+      "step": 340
+    },
+    {
+      "epoch": 0.6919560113678488,
+      "grad_norm": 0.6279313755216596,
+      "learning_rate": 1.7960038004162426e-05,
+      "loss": 0.4048,
+      "step": 350
+    },
+    {
+      "epoch": 0.7117261831212158,
+      "grad_norm": 0.6685300488229989,
+      "learning_rate": 1.7828780423749536e-05,
+      "loss": 0.4075,
+      "step": 360
+    },
+    {
+      "epoch": 0.731496354874583,
+      "grad_norm": 0.574309905920571,
+      "learning_rate": 1.769394242548469e-05,
+      "loss": 0.4102,
+      "step": 370
+    },
+    {
+      "epoch": 0.7512665266279501,
+      "grad_norm": 0.5657263244028015,
+      "learning_rate": 1.755558567623678e-05,
+      "loss": 0.396,
+      "step": 380
+    },
+    {
+      "epoch": 0.7710366983813172,
+      "grad_norm": 0.5797623404841392,
+      "learning_rate": 1.741377345214184e-05,
+      "loss": 0.4063,
+      "step": 390
+    },
+    {
+      "epoch": 0.7908068701346843,
+      "grad_norm": 0.5734761888945618,
+      "learning_rate": 1.7268570609664253e-05,
+      "loss": 0.4066,
+      "step": 400
+    },
+    {
+      "epoch": 0.8105770418880514,
+      "grad_norm": 0.5725087962478311,
+      "learning_rate": 1.71200435559353e-05,
+      "loss": 0.3848,
+      "step": 410
+    },
+    {
+      "epoch": 0.8303472136414185,
+      "grad_norm": 0.6574852860839745,
+      "learning_rate": 1.6968260218382484e-05,
+      "loss": 0.3882,
+      "step": 420
+    },
+    {
+      "epoch": 0.8501173853947857,
+      "grad_norm": 0.6526897086539006,
+      "learning_rate": 1.6813290013663573e-05,
+      "loss": 0.3957,
+      "step": 430
+    },
+    {
+      "epoch": 0.8698875571481527,
+      "grad_norm": 0.6101372561347892,
+      "learning_rate": 1.665520381591955e-05,
+      "loss": 0.3958,
+      "step": 440
+    },
+    {
+      "epoch": 0.8896577289015198,
+      "grad_norm": 0.6102719326578399,
+      "learning_rate": 1.6494073924361012e-05,
+      "loss": 0.3846,
+      "step": 450
+    },
+    {
+      "epoch": 0.909427900654887,
+      "grad_norm": 0.6284095546066177,
+      "learning_rate": 1.6329974030202803e-05,
+      "loss": 0.3925,
+      "step": 460
+    },
+    {
+      "epoch": 0.929198072408254,
+      "grad_norm": 0.5573501907726315,
+      "learning_rate": 1.6162979182962068e-05,
+      "loss": 0.3806,
+      "step": 470
+    },
+    {
+      "epoch": 0.9489682441616212,
+      "grad_norm": 0.6587508652316405,
+      "learning_rate": 1.5993165756135035e-05,
+      "loss": 0.4075,
+      "step": 480
+    },
+    {
+      "epoch": 0.9687384159149882,
+      "grad_norm": 0.5309276250787578,
+      "learning_rate": 1.5820611412268366e-05,
+      "loss": 0.3771,
+      "step": 490
+    },
+    {
+      "epoch": 0.9885085876683554,
+      "grad_norm": 0.5555801576940601,
+      "learning_rate": 1.564539506744091e-05,
+      "loss": 0.3869,
+      "step": 500
+    },
+    {
+      "epoch": 0.9885085876683554,
+      "eval_loss": 0.39637455344200134,
+      "eval_runtime": 81.285,
+      "eval_samples_per_second": 83.841,
+      "eval_steps_per_second": 5.241,
+      "step": 500
+    },
+    {
+      "epoch": 1.0082787594217224,
+      "grad_norm": 0.5067307545314473,
+      "learning_rate": 1.546759685517219e-05,
+      "loss": 0.356,
+      "step": 510
+    },
+    {
+      "epoch": 1.0280489311750896,
+      "grad_norm": 0.5618999195126448,
+      "learning_rate": 1.528729808977412e-05,
+      "loss": 0.3228,
+      "step": 520
+    },
+    {
+      "epoch": 1.0478191029284567,
+      "grad_norm": 0.5098273070317775,
+      "learning_rate": 1.5104581229162674e-05,
+      "loss": 0.3182,
+      "step": 530
+    },
+    {
+      "epoch": 1.0675892746818239,
+      "grad_norm": 0.5533254988756643,
+      "learning_rate": 1.4919529837146529e-05,
+      "loss": 0.3171,
+      "step": 540
+    },
+    {
+      "epoch": 1.087359446435191,
+      "grad_norm": 0.5964907411663062,
+      "learning_rate": 1.4732228545209993e-05,
+      "loss": 0.3202,
+      "step": 550
+    },
+    {
+      "epoch": 1.107129618188558,
+      "grad_norm": 0.5065864215097721,
+      "learning_rate": 1.4542763013807568e-05,
+      "loss": 0.3121,
+      "step": 560
+    },
+    {
+      "epoch": 1.126899789941925,
+      "grad_norm": 0.5343714773450696,
+      "learning_rate": 1.4351219893187984e-05,
+      "loss": 0.3179,
+      "step": 570
+    },
+    {
+      "epoch": 1.1466699616952922,
+      "grad_norm": 0.5681605326118392,
+      "learning_rate": 1.4157686783765534e-05,
+      "loss": 0.3112,
+      "step": 580
+    },
+    {
+      "epoch": 1.1664401334486594,
+      "grad_norm": 0.5604875931807674,
+      "learning_rate": 1.3962252196056847e-05,
+      "loss": 0.32,
+      "step": 590
+    },
+    {
+      "epoch": 1.1862103052020265,
+      "grad_norm": 0.5551583789751445,
+      "learning_rate": 1.3765005510201466e-05,
+      "loss": 0.3156,
+      "step": 600
+    },
+    {
+      "epoch": 1.2059804769553935,
+      "grad_norm": 0.5770289171897912,
+      "learning_rate": 1.3566036935084692e-05,
+      "loss": 0.3196,
+      "step": 610
+    },
+    {
+      "epoch": 1.2257506487087606,
+      "grad_norm": 0.5283055302681036,
+      "learning_rate": 1.3365437467081406e-05,
+      "loss": 0.3218,
+      "step": 620
+    },
+    {
+      "epoch": 1.2455208204621278,
+      "grad_norm": 0.5056609484631435,
+      "learning_rate": 1.3163298848439758e-05,
+      "loss": 0.315,
+      "step": 630
+    },
+    {
+      "epoch": 1.265290992215495,
+      "grad_norm": 0.555127018751701,
+      "learning_rate": 1.2959713525323725e-05,
+      "loss": 0.3221,
+      "step": 640
+    },
+    {
+      "epoch": 1.285061163968862,
+      "grad_norm": 0.5130249143640312,
+      "learning_rate": 1.2754774605533728e-05,
+      "loss": 0.3111,
+      "step": 650
+    },
+    {
+      "epoch": 1.3048313357222292,
+      "grad_norm": 0.519020033436049,
+      "learning_rate": 1.2548575815924689e-05,
+      "loss": 0.3151,
+      "step": 660
+    },
+    {
+      "epoch": 1.3246015074755961,
+      "grad_norm": 0.5149083765226259,
+      "learning_rate": 1.234121145954094e-05,
+      "loss": 0.31,
+      "step": 670
+    },
+    {
+      "epoch": 1.3443716792289633,
+      "grad_norm": 0.5301240336350007,
+      "learning_rate": 1.2132776372487634e-05,
+      "loss": 0.3139,
+      "step": 680
+    },
+    {
+      "epoch": 1.3641418509823304,
+      "grad_norm": 0.4880188656600289,
+      "learning_rate": 1.1923365880558345e-05,
+      "loss": 0.3147,
+      "step": 690
+    },
+    {
+      "epoch": 1.3839120227356976,
+      "grad_norm": 0.5098313222505657,
+      "learning_rate": 1.1713075755638748e-05,
+      "loss": 0.3153,
+      "step": 700
+    },
+    {
+      "epoch": 1.4036821944890647,
+      "grad_norm": 0.5563900244184181,
+      "learning_rate": 1.1502002171906248e-05,
+      "loss": 0.3137,
+      "step": 710
+    },
+    {
+      "epoch": 1.4234523662424317,
+      "grad_norm": 0.5308336365319577,
+      "learning_rate": 1.1290241661845649e-05,
+      "loss": 0.301,
+      "step": 720
+    },
+    {
+      "epoch": 1.4432225379957988,
+      "grad_norm": 0.5347937895279813,
+      "learning_rate": 1.1077891072100944e-05,
+      "loss": 0.3114,
+      "step": 730
+    },
+    {
+      "epoch": 1.462992709749166,
+      "grad_norm": 0.5353226891153153,
+      "learning_rate": 1.0865047519183411e-05,
+      "loss": 0.3132,
+      "step": 740
+    },
+    {
+      "epoch": 1.482762881502533,
+      "grad_norm": 0.5827716787607696,
+      "learning_rate": 1.065180834505634e-05,
+      "loss": 0.3032,
+      "step": 750
+    },
+    {
+      "epoch": 1.5025330532559003,
+      "grad_norm": 0.5898123869988975,
+      "learning_rate": 1.0438271072616584e-05,
+      "loss": 0.3113,
+      "step": 760
+    },
+    {
+      "epoch": 1.5223032250092672,
+      "grad_norm": 0.5298973225294878,
+      "learning_rate": 1.022453336109341e-05,
+      "loss": 0.3219,
+      "step": 770
+    },
+    {
+      "epoch": 1.5420733967626343,
+      "grad_norm": 0.5690896784690122,
+      "learning_rate": 1.001069296138499e-05,
+      "loss": 0.3134,
+      "step": 780
+    },
+    {
+      "epoch": 1.5618435685160015,
+      "grad_norm": 0.5827223568466493,
+      "learning_rate": 9.796847671352948e-06,
+      "loss": 0.3126,
+      "step": 790
+    },
+    {
+      "epoch": 1.5816137402693686,
+      "grad_norm": 0.5429937702192498,
+      "learning_rate": 9.583095291095454e-06,
+      "loss": 0.3039,
+      "step": 800
+    },
+    {
+      "epoch": 1.6013839120227358,
+      "grad_norm": 0.5498582424119081,
+      "learning_rate": 9.36953357821928e-06,
+      "loss": 0.3043,
+      "step": 810
+    },
+    {
+      "epoch": 1.6211540837761027,
+      "grad_norm": 0.4994859445298306,
+      "learning_rate": 9.156260203131299e-06,
+      "loss": 0.296,
+      "step": 820
+    },
+    {
+      "epoch": 1.64092425552947,
+      "grad_norm": 0.5154683199933622,
+      "learning_rate": 8.943372704369844e-06,
+      "loss": 0.2962,
+      "step": 830
+    },
+    {
+      "epoch": 1.660694427282837,
+      "grad_norm": 0.5167087299820147,
+      "learning_rate": 8.730968443996411e-06,
+      "loss": 0.3129,
+      "step": 840
+    },
+    {
+      "epoch": 1.6804645990362042,
+      "grad_norm": 0.5490107000824208,
+      "learning_rate": 8.519144563068037e-06,
+      "loss": 0.3119,
+      "step": 850
+    },
+    {
+      "epoch": 1.7002347707895713,
+      "grad_norm": 0.540920173955098,
+      "learning_rate": 8.30799793721077e-06,
+      "loss": 0.3056,
+      "step": 860
+    },
+    {
+      "epoch": 1.7200049425429382,
+      "grad_norm": 0.5237649749912278,
+      "learning_rate": 8.097625132314529e-06,
+      "loss": 0.311,
+      "step": 870
+    },
+    {
+      "epoch": 1.7397751142963056,
+      "grad_norm": 0.6272718536197235,
+      "learning_rate": 7.888122360369617e-06,
+      "loss": 0.3047,
+      "step": 880
+    },
+    {
+      "epoch": 1.7595452860496725,
+      "grad_norm": 0.5296663362064814,
+      "learning_rate": 7.67958543546509e-06,
+      "loss": 0.3011,
+      "step": 890
+    },
+    {
+      "epoch": 1.7793154578030397,
+      "grad_norm": 0.5233109909144185,
+      "learning_rate": 7.472109729969097e-06,
+      "loss": 0.2958,
+      "step": 900
+    },
+    {
+      "epoch": 1.7990856295564068,
+      "grad_norm": 0.5025706432729875,
+      "learning_rate": 7.2657901309112325e-06,
+      "loss": 0.3035,
+      "step": 910
+    },
+    {
+      "epoch": 1.8188558013097738,
+      "grad_norm": 0.5402559569074855,
+      "learning_rate": 7.060720996586878e-06,
+      "loss": 0.3134,
+      "step": 920
+    },
+    {
+      "epoch": 1.8386259730631411,
+      "grad_norm": 0.547259180262811,
+      "learning_rate": 6.85699611340333e-06,
+      "loss": 0.3055,
+      "step": 930
+    },
+    {
+      "epoch": 1.858396144816508,
+      "grad_norm": 0.5197703756039205,
+      "learning_rate": 6.654708652987475e-06,
+      "loss": 0.3043,
+      "step": 940
+    },
+    {
+      "epoch": 1.8781663165698752,
+      "grad_norm": 0.5323204838924858,
+      "learning_rate": 6.453951129574644e-06,
+      "loss": 0.3048,
+      "step": 950
+    },
+    {
+      "epoch": 1.8979364883232424,
+      "grad_norm": 0.47841296810426304,
+      "learning_rate": 6.254815357698118e-06,
+      "loss": 0.3006,
+      "step": 960
+    },
+    {
+      "epoch": 1.9177066600766093,
+      "grad_norm": 0.5083748369122659,
+      "learning_rate": 6.05739241019862e-06,
+      "loss": 0.3025,
+      "step": 970
+    },
+    {
+      "epoch": 1.9374768318299767,
+      "grad_norm": 0.4928502314870966,
+      "learning_rate": 5.861772576573028e-06,
+      "loss": 0.2956,
+      "step": 980
+    },
+    {
+      "epoch": 1.9572470035833436,
+      "grad_norm": 0.48791550019041724,
+      "learning_rate": 5.668045321681328e-06,
+      "loss": 0.3091,
+      "step": 990
+    },
+    {
+      "epoch": 1.9770171753367107,
+      "grad_norm": 0.5066742151391088,
+      "learning_rate": 5.476299244830734e-06,
+      "loss": 0.2999,
+      "step": 1000
+    },
+    {
+      "epoch": 1.9770171753367107,
+      "eval_loss": 0.3708229959011078,
+      "eval_runtime": 81.8274,
+      "eval_samples_per_second": 83.285,
+      "eval_steps_per_second": 5.206,
+      "step": 1000
+    },
+    {
+      "epoch": 1.9967873470900779,
+      "grad_norm": 0.5294994920674526,
+      "learning_rate": 5.28662203925562e-06,
+      "loss": 0.3066,
+      "step": 1010
+    },
+    {
+      "epoch": 2.016557518843445,
+      "grad_norm": 0.5249670268523262,
+      "learning_rate": 5.099100452011876e-06,
+      "loss": 0.2574,
+      "step": 1020
+    },
+    {
+      "epoch": 2.036327690596812,
+      "grad_norm": 0.5209226672918731,
+      "learning_rate": 4.9138202443039625e-06,
+      "loss": 0.2297,
+      "step": 1030
+    },
+    {
+      "epoch": 2.056097862350179,
+      "grad_norm": 0.4884858081963297,
+      "learning_rate": 4.730866152262841e-06,
+      "loss": 0.2445,
+      "step": 1040
+    },
+    {
+      "epoch": 2.0758680341035465,
+      "grad_norm": 0.4742867737071898,
+      "learning_rate": 4.550321848192731e-06,
+      "loss": 0.2505,
+      "step": 1050
+    },
+    {
+      "epoch": 2.0956382058569134,
+      "grad_norm": 0.522161636411317,
+      "learning_rate": 4.3722699023043634e-06,
+      "loss": 0.2494,
+      "step": 1060
+    },
+    {
+      "epoch": 2.1154083776102803,
+      "grad_norm": 0.4240817050624604,
+      "learning_rate": 4.196791744952291e-06,
+      "loss": 0.239,
+      "step": 1070
+    },
+    {
+      "epoch": 2.1351785493636477,
+      "grad_norm": 0.5016013699913245,
+      "learning_rate": 4.023967629393498e-06,
+      "loss": 0.2463,
+      "step": 1080
+    },
+    {
+      "epoch": 2.1549487211170146,
+      "grad_norm": 0.4907508903718958,
+      "learning_rate": 3.853876595084332e-06,
+      "loss": 0.2455,
+      "step": 1090
+    },
+    {
+      "epoch": 2.174718892870382,
+      "grad_norm": 0.4844801070690468,
+      "learning_rate": 3.68659643153258e-06,
+      "loss": 0.2406,
+      "step": 1100
+    },
+    {
+      "epoch": 2.194489064623749,
+      "grad_norm": 0.4805850138516767,
+      "learning_rate": 3.5222036427211704e-06,
+      "loss": 0.2333,
+      "step": 1110
+    },
+    {
+      "epoch": 2.214259236377116,
+      "grad_norm": 0.4850837241450545,
+      "learning_rate": 3.360773412119801e-06,
+      "loss": 0.2397,
+      "step": 1120
+    },
+    {
+      "epoch": 2.2340294081304832,
+      "grad_norm": 0.46778943159588726,
+      "learning_rate": 3.202379568300511e-06,
+      "loss": 0.2394,
+      "step": 1130
+    },
+    {
+      "epoch": 2.25379957988385,
+      "grad_norm": 0.4262009004075529,
+      "learning_rate": 3.0470945511728856e-06,
+      "loss": 0.2319,
+      "step": 1140
+    },
+    {
+      "epoch": 2.2735697516372175,
+      "grad_norm": 0.4910291271379484,
+      "learning_rate": 2.8949893788543505e-06,
+      "loss": 0.2461,
+      "step": 1150
+    },
+    {
+      "epoch": 2.2933399233905845,
+      "grad_norm": 0.5099593229914312,
+      "learning_rate": 2.74613361519073e-06,
+      "loss": 0.2391,
+      "step": 1160
+    },
+    {
+      "epoch": 2.3131100951439514,
+      "grad_norm": 0.47785223375797403,
+      "learning_rate": 2.60059533794187e-06,
+      "loss": 0.2532,
+      "step": 1170
+    },
+    {
+      "epoch": 2.3328802668973188,
+      "grad_norm": 0.48894964709263233,
+      "learning_rate": 2.458441107646935e-06,
+      "loss": 0.2442,
+      "step": 1180
+    },
+    {
+      "epoch": 2.3526504386506857,
+      "grad_norm": 0.4887403938640547,
+      "learning_rate": 2.3197359371835802e-06,
+      "loss": 0.2374,
+      "step": 1190
+    },
+    {
+      "epoch": 2.372420610404053,
+      "grad_norm": 0.5094057476841393,
+      "learning_rate": 2.1845432620349338e-06,
+      "loss": 0.2383,
+      "step": 1200
+    },
+    {
+      "epoch": 2.39219078215742,
+      "grad_norm": 0.4810409572583604,
+      "learning_rate": 2.0529249112779824e-06,
+      "loss": 0.2388,
+      "step": 1210
+    },
+    {
+      "epoch": 2.411960953910787,
+      "grad_norm": 0.506178462808245,
+      "learning_rate": 1.924941079306647e-06,
+      "loss": 0.2437,
+      "step": 1220
+    },
+    {
+      "epoch": 2.4317311256641543,
+      "grad_norm": 0.5101355373820005,
+      "learning_rate": 1.8006502983024332e-06,
+      "loss": 0.2482,
+      "step": 1230
+    },
+    {
+      "epoch": 2.451501297417521,
+      "grad_norm": 0.47636683086858184,
+      "learning_rate": 1.6801094114653082e-06,
+      "loss": 0.2313,
+      "step": 1240
+    },
+    {
+      "epoch": 2.4712714691708886,
+      "grad_norm": 0.43908801679241943,
+      "learning_rate": 1.5633735470170108e-06,
+      "loss": 0.2403,
+      "step": 1250
+    },
+    {
+      "epoch": 2.4910416409242555,
+      "grad_norm": 0.46494594811457074,
+      "learning_rate": 1.4504960929886746e-06,
+      "loss": 0.2442,
+      "step": 1260
+    },
+    {
+      "epoch": 2.5108118126776224,
+      "grad_norm": 0.49463377523743013,
+      "learning_rate": 1.3415286728043374e-06,
+      "loss": 0.239,
+      "step": 1270
+    },
+    {
+      "epoch": 2.53058198443099,
+      "grad_norm": 0.46074492651979493,
+      "learning_rate": 1.2365211216714523e-06,
+      "loss": 0.2385,
+      "step": 1280
+    },
+    {
+      "epoch": 2.5503521561843567,
+      "grad_norm": 0.5218071815372649,
+      "learning_rate": 1.135521463789232e-06,
+      "loss": 0.2482,
+      "step": 1290
+    },
+    {
+      "epoch": 2.570122327937724,
+      "grad_norm": 0.4982910156071213,
+      "learning_rate": 1.0385758903852406e-06,
+      "loss": 0.2381,
+      "step": 1300
+    },
+    {
+      "epoch": 2.589892499691091,
+      "grad_norm": 0.5007115279363157,
+      "learning_rate": 9.457287385902836e-07,
+      "loss": 0.2405,
+      "step": 1310
+    },
+    {
+      "epoch": 2.6096626714444584,
+      "grad_norm": 0.4830944811979268,
+      "learning_rate": 8.570224711612385e-07,
+      "loss": 0.2304,
+      "step": 1320
+    },
+    {
+      "epoch": 2.6294328431978253,
+      "grad_norm": 0.47077203411861107,
+      "learning_rate": 7.724976570611254e-07,
+      "loss": 0.2259,
+      "step": 1330
+    },
+    {
+      "epoch": 2.6492030149511923,
+      "grad_norm": 0.49030170297745246,
+      "learning_rate": 6.921929529052729e-07,
+      "loss": 0.2318,
+      "step": 1340
+    },
+    {
+      "epoch": 2.6689731867045596,
+      "grad_norm": 0.49483419009773855,
+      "learning_rate": 6.161450852820872e-07,
+      "loss": 0.2377,
+      "step": 1350
+    },
+    {
+      "epoch": 2.6887433584579266,
+      "grad_norm": 0.5047141018868604,
+      "learning_rate": 5.443888339565018e-07,
+      "loss": 0.2402,
+      "step": 1360
+    },
+    {
+      "epoch": 2.7085135302112935,
+      "grad_norm": 0.4627659592534385,
+      "learning_rate": 4.769570159637782e-07,
+      "loss": 0.2401,
+      "step": 1370
+    },
+    {
+      "epoch": 2.728283701964661,
+      "grad_norm": 0.5051673776560596,
+      "learning_rate": 4.138804706009547e-07,
+      "loss": 0.2306,
+      "step": 1380
+    },
+    {
+      "epoch": 2.748053873718028,
+      "grad_norm": 0.4697753055417383,
+      "learning_rate": 3.5518804532278736e-07,
+      "loss": 0.2407,
+      "step": 1390
+    },
+    {
+      "epoch": 2.767824045471395,
+      "grad_norm": 0.48255171108333217,
+      "learning_rate": 3.0090658254864013e-07,
+      "loss": 0.227,
+      "step": 1400
+    },
+    {
+      "epoch": 2.787594217224762,
+      "grad_norm": 0.48725984214652784,
+      "learning_rate": 2.5106090738636014e-07,
+      "loss": 0.2306,
+      "step": 1410
+    },
+    {
+      "epoch": 2.8073643889781295,
+      "grad_norm": 0.5176868460783628,
+      "learning_rate": 2.0567381627875639e-07,
+      "loss": 0.242,
+      "step": 1420
+    },
+    {
+      "epoch": 2.8271345607314964,
+      "grad_norm": 0.486977389827133,
+      "learning_rate": 1.6476606657786386e-07,
+      "loss": 0.2309,
+      "step": 1430
+    },
+    {
+      "epoch": 2.8469047324848633,
+      "grad_norm": 0.48480851362499594,
+      "learning_rate": 1.2835636705176403e-07,
+      "loss": 0.2287,
+      "step": 1440
+    },
+    {
+      "epoch": 2.8666749042382307,
+      "grad_norm": 0.506776005156481,
+      "learning_rate": 9.64613693283123e-08,
+      "loss": 0.2469,
+      "step": 1450
+    },
+    {
+      "epoch": 2.8864450759915976,
+      "grad_norm": 0.5370285292991429,
+      "learning_rate": 6.909566027966996e-08,
+      "loss": 0.2458,
+      "step": 1460
+    },
+    {
+      "epoch": 2.9062152477449645,
+      "grad_norm": 0.46946884718156673,
+      "learning_rate": 4.6271755351144923e-08,
+      "loss": 0.2456,
+      "step": 1470
+    },
+    {
+      "epoch": 2.925985419498332,
+      "grad_norm": 0.47309149600612144,
+      "learning_rate": 2.8000092837368042e-08,
+      "loss": 0.2413,
+      "step": 1480
+    },
+    {
+      "epoch": 2.945755591251699,
+      "grad_norm": 0.48781257822426394,
+      "learning_rate": 1.4289029108436325e-08,
+      "loss": 0.2388,
+      "step": 1490
+    },
+    {
+      "epoch": 2.965525763005066,
+      "grad_norm": 0.4929015642135056,
+      "learning_rate": 5.14483478821437e-09,
+      "loss": 0.2454,
+      "step": 1500
+    },
+    {
+      "epoch": 2.965525763005066,
+      "eval_loss": 0.37777501344680786,
+      "eval_runtime": 82.1049,
+      "eval_samples_per_second": 83.004,
+      "eval_steps_per_second": 5.188,
+      "step": 1500
+    },
+    {
+      "epoch": 2.985295934758433,
+      "grad_norm": 0.49492433015567533,
+      "learning_rate": 5.716918865195098e-10,
+      "loss": 0.2349,
+      "step": 1510
+    },
+    {
+      "epoch": 2.995181020635117,
+      "step": 1515,
+      "total_flos": 504886288187392.0,
+      "train_loss": 0.33701497725527674,
+      "train_runtime": 18615.961,
+      "train_samples_per_second": 20.865,
+      "train_steps_per_second": 0.081
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1515,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 504886288187392.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

added_tokens.json ADDED Viewed

	@@ -0,0 +1,28 @@

+{
+  "</chain_of_thought>": 151666,
+  "</cite>": 151668,
+  "</tool_call>": 151658,
+  "<chain_of_thought>": 151665,
+  "<cite docs=": 151667,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json ADDED Viewed

	@@ -0,0 +1,12 @@

+{
+    "epoch": 2.995181020635117,
+    "eval_loss": 0.37776756286621094,
+    "eval_runtime": 82.0824,
+    "eval_samples_per_second": 83.026,
+    "eval_steps_per_second": 5.19,
+    "total_flos": 504886288187392.0,
+    "train_loss": 0.33701497725527674,
+    "train_runtime": 18615.961,
+    "train_samples_per_second": 20.865,
+    "train_steps_per_second": 0.081
+}

config.json ADDED Viewed

	@@ -0,0 +1,35 @@

+{
+  "_name_or_path": "models/OmegaNeo-8b",
+  "architectures": [
+    "OmegaNeoForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_omeganeo.OmegaNeoConfig",
+    "AutoModel": "configuration_omeganeo.OmegaNeoModel",
+    "AutoModelForCausalLM": "modeling_omeganeo.OmegaNeoForCausalLM"
+  },
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "omeganeo",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.0",
+  "use_cache": false,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

configuration_omeganeo.py ADDED Viewed

	@@ -0,0 +1,59 @@

+"""OmegaNeo model configuration"""
+from transformers import PretrainedConfig
+from transformers.utils import logging
+logger = logging.get_logger(__name__)
+class OmegaNeoConfig(PretrainedConfig):
+    model_type = "omeganeo"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    def __init__(
+        self,
+        vocab_size=152064,
+        hidden_size=3584,
+        intermediate_size=18944,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=32,
+        hidden_act="silu",
+        max_position_embeddings=32768,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        use_sliding_window=False,
+        sliding_window=4096,
+        max_window_layers=28,
+        attention_dropout=0.0,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.use_sliding_window = use_sliding_window
+        self.sliding_window = sliding_window
+        self.max_window_layers = max_window_layers
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.attention_dropout = attention_dropout
+        super().__init__(
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

eval_results.json ADDED Viewed

	@@ -0,0 +1,7 @@

+{
+    "epoch": 2.995181020635117,
+    "eval_loss": 0.37776756286621094,
+    "eval_runtime": 82.0824,
+    "eval_samples_per_second": 83.026,
+    "eval_steps_per_second": 5.19
+}

generation_config.json ADDED Viewed

	@@ -0,0 +1,6 @@

+{
+  "_from_model_config": true,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "transformers_version": "4.45.0"
+}

logs.jsonl ADDED Viewed

The diff for this file is too large to render. See raw diff

merges.txt ADDED Viewed

The diff for this file is too large to render. See raw diff

model-00001-of-00004.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:dc8837deba5f9fa9f352628a688c074d5b4a488cbfa47eebc08532e98559cbf2
+size 4877660776

model-00002-of-00004.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:15971ca2751e9735417cec1a68d493870f359f00c801c9c3ac77617bd784ff80
+size 4932751008

model-00003-of-00004.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:111986292557ce995625b1140440d0ebef5978eac4c8e5493f9d002380f19536
+size 4991495888

model-00004-of-00004.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:132be700dc2ca6a0fdc24bed26b9c6626baad83ec20dcaf2f2ccaf04634e52c0
+size 2293832048

model.safetensors.index.json ADDED Viewed

	@@ -0,0 +1,394 @@

+{
+  "metadata": {
+    "total_size": 17095695360
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

modeling_omeganeo.py ADDED Viewed

	@@ -0,0 +1,1343 @@

+""" PyTorch OmegaNeo model."""
+import inspect
+import math
+import warnings
+from typing import List, Optional, Tuple, Union
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.modeling_attn_mask_utils import _prepare_4d_causal_attention_mask, _prepare_4d_causal_attention_mask_for_sdpa
+from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, SequenceClassifierOutputWithPast
+from transformers.modeling_utils import PreTrainedModel
+from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS
+from transformers.utils import (
+    add_start_docstrings,
+    add_start_docstrings_to_model_forward,
+    is_flash_attn_2_available,
+    is_flash_attn_greater_or_equal_2_10,
+    logging,
+    replace_return_docstrings,
+)
+from .configuration_omeganeo import OmegaNeoConfig
+if is_flash_attn_2_available():
+    from flash_attn import flash_attn_func, flash_attn_varlen_func
+    from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input  # noqa
+    _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters)
+logger = logging.get_logger(__name__)
+_CHECKPOINT_FOR_DOC = "/models/omega_neo_small_chat"
+_CONFIG_FOR_DOC = "OmegaNeoConfig"
+def _get_unpad_data(attention_mask):
+    seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
+    indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
+    max_seqlen_in_batch = seqlens_in_batch.max().item()
+    cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0))
+    return (
+        indices,
+        cu_seqlens,
+        max_seqlen_in_batch,
+    )
+class OmegaNeoRMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        """
+        OmegaNeoRMSNorm is equivalent to T5LayerNorm
+        """
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+class OmegaNeoRotaryEmbedding(nn.Module):
+    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
+        super().__init__()
+        self.dim = dim
+        self.max_position_embeddings = max_position_embeddings
+        self.base = base
+        inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float().to(device) / self.dim))
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        # Build here to make `torch.jit.trace` work.
+        self._set_cos_sin_cache(
+            seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype()
+        )
+    def _set_cos_sin_cache(self, seq_len, device, dtype):
+        self.max_seq_len_cached = seq_len
+        t = torch.arange(self.max_seq_len_cached, device=device, dtype=torch.int64).type_as(self.inv_freq)
+        freqs = torch.outer(t, self.inv_freq)
+        # Different from paper, but it uses a different permutation in order to obtain the same calculation
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
+        self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
+    def forward(self, x, seq_len=None):
+        # x: [bs, num_attention_heads, seq_len, head_size]
+        if seq_len > self.max_seq_len_cached:
+            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype)
+        return (
+            self.cos_cached[:seq_len].to(dtype=x.dtype),
+            self.sin_cached[:seq_len].to(dtype=x.dtype),
+        )
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1):
+    """Applies Rotary Position Embedding to the query and key tensors.
+    Args:
+        q (`torch.Tensor`): The query tensor.
+        k (`torch.Tensor`): The key tensor.
+        cos (`torch.Tensor`): The cosine part of the rotary embedding.
+        sin (`torch.Tensor`): The sine part of the rotary embedding.
+        position_ids (`torch.Tensor`):
+            The position indices of the tokens corresponding to the query and key tensors. For example, this can be
+            used to pass offsetted position ids when working with a KV-cache.
+        unsqueeze_dim (`int`, *optional*, defaults to 1):
+            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
+            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
+            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
+            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
+            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
+            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
+    Returns:
+        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
+    """
+    cos = cos[position_ids].unsqueeze(unsqueeze_dim)
+    sin = sin[position_ids].unsqueeze(unsqueeze_dim)
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+class OmegaNeoMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = config.intermediate_size
+        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
+        self.act_fn = ACT2FN[config.hidden_act]
+    def forward(self, x):
+        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """
+    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+    """
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+class OmegaNeoAttention(nn.Module):
+    """
+    Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer
+    and "Generating Long Sequences with Sparse Transformers".
+    """
+    def __init__(self, config: OmegaNeoConfig, layer_idx: Optional[int] = None):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        if layer_idx is None:
+            logger.warning_once(
+                f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will "
+                "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` "
+                "when creating this class."
+            )
+        self.hidden_size = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.hidden_size // self.num_heads
+        self.num_key_value_heads = config.num_key_value_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+        self.max_position_embeddings = config.max_position_embeddings
+        self.rope_theta = config.rope_theta
+        self.is_causal = True
+        self.attention_dropout = config.attention_dropout
+        if (self.head_dim * self.num_heads) != self.hidden_size:
+            raise ValueError(
+                f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
+                f" and `num_heads`: {self.num_heads})."
+            )
+        self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=True)
+        self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True)
+        self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True)
+        self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
+        self.rotary_emb = OmegaNeoRotaryEmbedding(
+            self.head_dim,
+            max_position_embeddings=self.max_position_embeddings,
+            base=self.rope_theta,
+        )
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        **kwargs,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        if "padding_mask" in kwargs:
+            warnings.warn(
+                "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"
+            )
+        bsz, q_len, _ = hidden_states.size()
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            if self.layer_idx is None:
+                raise ValueError(
+                    f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} "
+                    "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "
+                    "with a layer index."
+                )
+            kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
+        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+        if past_key_value is not None:
+            cache_kwargs = {"sin": sin, "cos": cos}  # Specific to RoPE models
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+        # repeat k/v heads if n_kv_heads < n_heads
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
+        if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
+            raise ValueError(
+                f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+        if attention_mask is not None:
+            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+                )
+            attn_weights = attn_weights + attention_mask
+        # upcast attention to fp32
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)
+        attn_output = torch.matmul(attn_weights, value_states)
+        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
+        attn_output = self.o_proj(attn_output)
+        if not output_attentions:
+            attn_weights = None
+        return attn_output, attn_weights, past_key_value
+class OmegaNeoFlashAttention2(OmegaNeoAttention):
+    """
+    OmegaNeo flash attention module, following OmegaNeo attention module. This module inherits from `OmegaNeoAttention`
+    as the weights of the module stays untouched. The only required change would be on the forward pass
+    where it needs to correctly call the public API of flash attention and deal with padding tokens
+    in case the input contains any of them. Additionally, for sliding window attention, we apply SWA only to the bottom
+    config.max_window_layers layers.
+    """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1.
+        # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0.
+        # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left).
+        self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        **kwargs,
+    ):
+        if "padding_mask" in kwargs:
+            warnings.warn(
+                "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"
+            )
+            # overwrite attention_mask with padding_mask
+            attention_mask = kwargs.pop("padding_mask")
+        bsz, q_len, _ = hidden_states.size()
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            if self.layer_idx is None:
+                raise ValueError(
+                    f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} "
+                    "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "
+                    "with a layer index."
+                )
+            kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
+        # Because the input can be padded, the absolute sequence length depends on the max position id.
+        rotary_seq_len = max(kv_seq_len, position_ids[:, -1].max().item()) + 1
+        cos, sin = self.rotary_emb(value_states, seq_len=rotary_seq_len)
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+        use_sliding_windows = (
+            _flash_supports_window_size
+            and getattr(self.config, "sliding_window", None) is not None
+            and kv_seq_len > self.config.sliding_window
+            and self.config.use_sliding_window
+        )
+        if not _flash_supports_window_size:
+            logger.warning_once(
+                "The current flash attention version does not support sliding window attention, for a more memory efficient implementation"
+                " make sure to upgrade flash-attn library."
+            )
+        if past_key_value is not None:
+            # Activate slicing cache only if the config has a value `sliding_windows` attribute
+            cache_has_contents = past_key_value.get_seq_length(self.layer_idx) > 0
+            if (
+                getattr(self.config, "sliding_window", None) is not None
+                and kv_seq_len > self.config.sliding_window
+                and cache_has_contents
+            ):
+                slicing_tokens = 1 - self.config.sliding_window
+                past_key = past_key_value[self.layer_idx][0]
+                past_value = past_key_value[self.layer_idx][1]
+                past_key = past_key[:, :, slicing_tokens:, :].contiguous()
+                past_value = past_value[:, :, slicing_tokens:, :].contiguous()
+                if past_key.shape[-2] != self.config.sliding_window - 1:
+                    raise ValueError(
+                        f"past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1, head_dim`), got"
+                        f" {past_key.shape}"
+                    )
+                if attention_mask is not None:
+                    attention_mask = attention_mask[:, slicing_tokens:]
+                    attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1)
+            cache_kwargs = {"sin": sin, "cos": cos}  # Specific to RoPE models
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+        # repeat k/v heads if n_kv_heads < n_heads
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+        dropout_rate = 0.0 if not self.training else self.attention_dropout
+        # In PEFT, usually we cast the layer norms in float32 for training stability reasons
+        # therefore the input hidden states gets silently casted in float32. Hence, we need
+        # cast them back in float16 just to be sure everything works as expected.
+        input_dtype = query_states.dtype
+        if input_dtype == torch.float32:
+            if torch.is_autocast_enabled():
+                target_dtype = torch.get_autocast_gpu_dtype()
+            # Handle the case where the model is quantized
+            elif hasattr(self.config, "_pre_quantization_dtype"):
+                target_dtype = self.config._pre_quantization_dtype
+            else:
+                target_dtype = self.q_proj.weight.dtype
+            logger.warning_once(
+                f"The input hidden states seems to be silently casted in float32, this might be related to"
+                f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
+                f" {target_dtype}."
+            )
+            query_states = query_states.to(target_dtype)
+            key_states = key_states.to(target_dtype)
+            value_states = value_states.to(target_dtype)
+        # Reashape to the expected shape for Flash Attention
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+        attn_output = self._flash_attention_forward(
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            q_len,
+            dropout=dropout_rate,
+            use_sliding_windows=use_sliding_windows,
+        )
+        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()
+        attn_output = self.o_proj(attn_output)
+        if not output_attentions:
+            attn_weights = None
+        return attn_output, attn_weights, past_key_value
+    def _flash_attention_forward(
+        self,
+        query_states,
+        key_states,
+        value_states,
+        attention_mask,
+        query_length,
+        dropout=0.0,
+        softmax_scale=None,
+        use_sliding_windows=False,
+    ):
+        """
+        Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
+        first unpad the input, then computes the attention scores and pad the final attention scores.
+        Args:
+            query_states (`torch.Tensor`):
+                Input query states to be passed to Flash Attention API
+            key_states (`torch.Tensor`):
+                Input key states to be passed to Flash Attention API
+            value_states (`torch.Tensor`):
+                Input value states to be passed to Flash Attention API
+            attention_mask (`torch.Tensor`):
+                The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
+                position of padding tokens and 1 for the position of non-padding tokens.
+            dropout (`float`):
+                Attention dropout
+            softmax_scale (`float`, *optional*):
+                The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
+            use_sliding_windows (`bool`, *optional*):
+                Whether to activate sliding window attention.
+        """
+        if not self._flash_attn_uses_top_left_mask:
+            causal = self.is_causal
+        else:
+            # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__.
+            causal = self.is_causal and query_length != 1
+        # Decide whether to use SWA or not by layer index.
+        if use_sliding_windows and self.layer_idx >= self.config.max_window_layers:
+            use_sliding_windows = False
+        # Contains at least one padding token in the sequence
+        if attention_mask is not None:
+            batch_size = query_states.shape[0]
+            query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
+                query_states, key_states, value_states, attention_mask, query_length
+            )
+            cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+            max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+            if not use_sliding_windows:
+                attn_output_unpad = flash_attn_varlen_func(
+                    query_states,
+                    key_states,
+                    value_states,
+                    cu_seqlens_q=cu_seqlens_q,
+                    cu_seqlens_k=cu_seqlens_k,
+                    max_seqlen_q=max_seqlen_in_batch_q,
+                    max_seqlen_k=max_seqlen_in_batch_k,
+                    dropout_p=dropout,
+                    softmax_scale=softmax_scale,
+                    causal=causal,
+                )
+            else:
+                attn_output_unpad = flash_attn_varlen_func(
+                    query_states,
+                    key_states,
+                    value_states,
+                    cu_seqlens_q=cu_seqlens_q,
+                    cu_seqlens_k=cu_seqlens_k,
+                    max_seqlen_q=max_seqlen_in_batch_q,
+                    max_seqlen_k=max_seqlen_in_batch_k,
+                    dropout_p=dropout,
+                    softmax_scale=softmax_scale,
+                    causal=causal,
+                    window_size=(self.config.sliding_window, self.config.sliding_window),
+                )
+            attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
+        else:
+            if not use_sliding_windows:
+                attn_output = flash_attn_func(
+                    query_states,
+                    key_states,
+                    value_states,
+                    dropout,
+                    softmax_scale=softmax_scale,
+                    causal=causal,
+                )
+            else:
+                attn_output = flash_attn_func(
+                    query_states,
+                    key_states,
+                    value_states,
+                    dropout,
+                    softmax_scale=softmax_scale,
+                    causal=causal,
+                    window_size=(self.config.sliding_window, self.config.sliding_window),
+                )
+        return attn_output
+    def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
+        batch_size, kv_seq_len, num_heads, head_dim = key_layer.shape
+        # On the first iteration we need to properly re-create the padding mask
+        # by slicing it on the proper place
+        if kv_seq_len != attention_mask.shape[-1]:
+            attention_mask_num_tokens = attention_mask.shape[-1]
+            attention_mask = attention_mask[:, attention_mask_num_tokens - kv_seq_len :]
+        indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
+        key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k)
+        value_layer = index_first_axis(value_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k)
+        if query_length == kv_seq_len:
+            query_layer = index_first_axis(
+                query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k
+            )
+            cu_seqlens_q = cu_seqlens_k
+            max_seqlen_in_batch_q = max_seqlen_in_batch_k
+            indices_q = indices_k
+        elif query_length == 1:
+            max_seqlen_in_batch_q = 1
+            cu_seqlens_q = torch.arange(
+                batch_size + 1, dtype=torch.int32, device=query_layer.device
+            )  # There is a memcpy here, that is very bad.
+            indices_q = cu_seqlens_q[:-1]
+            query_layer = query_layer.squeeze(1)
+        else:
+            # The -q_len: slice assumes left padding.
+            attention_mask = attention_mask[:, -query_length:]
+            query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
+        return (
+            query_layer,
+            key_layer,
+            value_layer,
+            indices_q,
+            (cu_seqlens_q, cu_seqlens_k),
+            (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+        )
+class OmegaNeoSdpaAttention(OmegaNeoAttention):
+    """
+    OmegaNeo attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
+    `OmegaNeoAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
+    SDPA API.
+    """
+    # Adapted from OmegaNeoAttention.forward
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        if output_attentions:
+            # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented.
+            logger.warning_once(
+                "OmegaNeoModel is using OmegaNeoSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, "
+                'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
+            )
+            return super().forward(
+                hidden_states=hidden_states,
+                attention_mask=attention_mask,
+                position_ids=position_ids,
+                past_key_value=past_key_value,
+                output_attentions=output_attentions,
+                use_cache=use_cache,
+            )
+        bsz, q_len, _ = hidden_states.size()
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
+        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+        if past_key_value is not None:
+            cache_kwargs = {"sin": sin, "cos": cos}  # Specific to RoPE models
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+        if attention_mask is not None:
+            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+                )
+        # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask,
+        # Reference: https://github.com/pytorch/pytorch/issues/112577.
+        if query_states.device.type == "cuda" and attention_mask is not None:
+            query_states = query_states.contiguous()
+            key_states = key_states.contiguous()
+            value_states = value_states.contiguous()
+        attn_output = torch.nn.functional.scaled_dot_product_attention(
+            query_states,
+            key_states,
+            value_states,
+            attn_mask=attention_mask,
+            dropout_p=self.attention_dropout if self.training else 0.0,
+            # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1.
+            is_causal=self.is_causal and attention_mask is None and q_len > 1,
+        )
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.view(bsz, q_len, self.hidden_size)
+        attn_output = self.o_proj(attn_output)
+        return attn_output, None, past_key_value
+OMEGANEO_ATTENTION_CLASSES = {
+    "eager": OmegaNeoAttention,
+    "flash_attention_2": OmegaNeoFlashAttention2,
+    "sdpa": OmegaNeoSdpaAttention,
+}
+class OmegaNeoDecoderLayer(nn.Module):
+    def __init__(self, config: OmegaNeoConfig, layer_idx: int):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+        if config.use_sliding_window and config._attn_implementation != "flash_attention_2":
+            logger.warning_once(
+                f"Sliding Window Attention is enabled but not implemented for `{config._attn_implementation}`; "
+                "unexpected results may be encountered."
+            )
+        self.self_attn = OMEGANEO_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx)
+        self.mlp = OmegaNeoMLP(config)
+        self.input_layernorm = OmegaNeoRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.post_attention_layernorm = OmegaNeoRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+        **kwargs,
+    ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
+        if "padding_mask" in kwargs:
+            warnings.warn(
+                "Passing `padding_mask` is deprecated and will be removed in v4.37. "
+                "Please make sure use `attention_mask` instead.`"
+            )
+        """
+        Args:
+            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
+                `(batch, sequence_length)` where padding elements are indicated by 0.
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            use_cache (`bool`, *optional*):
+                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+                (see `past_key_values`).
+            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
+        """
+        residual = hidden_states
+        hidden_states = self.input_layernorm(hidden_states)
+        # Self Attention
+        hidden_states, self_attn_weights, present_key_value = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_value=past_key_value,
+            output_attentions=output_attentions,
+            use_cache=use_cache,
+        )
+        hidden_states = residual + hidden_states
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+        outputs = (hidden_states,)
+        if output_attentions:
+            outputs += (self_attn_weights,)
+        if use_cache:
+            outputs += (present_key_value,)
+        return outputs
+OMEGANEO_START_DOCSTRING = r"""
+    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+    etc.)
+    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+    and behavior.
+    Parameters:
+        config ([`OmegaNeoConfig`]):
+            Model configuration class with all the parameters of the model. Initializing with a config file does not
+            load the weights associated with the model, only the configuration. Check out the
+            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+@add_start_docstrings(
+    "The bare OmegaNeo Model outputting raw hidden-states without any specific head on top.",
+    OMEGANEO_START_DOCSTRING,
+)
+class OmegaNeoPreTrainedModel(PreTrainedModel):
+    config_class = OmegaNeoConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["OmegaNeoDecoderLayer"]
+    _skip_keys_device_placement = "past_key_values"
+    _supports_flash_attn_2 = True
+    _supports_sdpa = True
+    _supports_cache_class = True
+    def _init_weights(self, module):
+        std = self.config.initializer_range
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+OMEGANEO_INPUTS_DOCSTRING = r"""
+    Args:
+        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+            it.
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+            [What are input IDs?](../glossary#input-ids)
+        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+            [What are attention masks?](../glossary#attention-mask)
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+            If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
+            `past_key_values`).
+            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
+            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
+            information on the default strategy.
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+            config.n_positions - 1]`.
+            [What are position IDs?](../glossary#position-ids)
+        past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
+            Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
+            blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
+            returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.
+            Two formats are allowed:
+            - a [`~cache_utils.Cache`] instance;
+            - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
+            cache format.
+            The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
+            legacy cache format will be returned.
+            If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
+            have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
+            of shape `(batch_size, sequence_length)`.
+        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+            model's internal embedding lookup matrix.
+        use_cache (`bool`, *optional*):
+            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+            `past_key_values`).
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+@add_start_docstrings(
+    "The bare OmegaNeo Model outputting raw hidden-states without any specific head on top.",
+    OMEGANEO_START_DOCSTRING,
+)
+class OmegaNeoModel(OmegaNeoPreTrainedModel):
+    """
+    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`OmegaNeoDecoderLayer`]
+    Args:
+        config: OmegaNeoConfig
+    """
+    def __init__(self, config: OmegaNeoConfig):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.layers = nn.ModuleList(
+            [OmegaNeoDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+        )
+        self._attn_implementation = config._attn_implementation
+        self.norm = OmegaNeoRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.gradient_checkpointing = False
+        # Initialize weights and apply final processing
+        self.post_init()
+    def get_input_embeddings(self):
+        return self.embed_tokens
+    def set_input_embeddings(self, value):
+        self.embed_tokens = value
+    @add_start_docstrings_to_model_forward(OMEGANEO_INPUTS_DOCSTRING)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPast]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        # retrieve input_ids and inputs_embeds
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
+        elif input_ids is not None:
+            batch_size, seq_length = input_ids.shape
+        elif inputs_embeds is not None:
+            batch_size, seq_length, _ = inputs_embeds.shape
+        else:
+            raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                logger.warning_once(
+                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                )
+                use_cache = False
+        past_key_values_length = 0
+        if use_cache:
+            use_legacy_cache = not isinstance(past_key_values, Cache)
+            if use_legacy_cache:
+                past_key_values = DynamicCache.from_legacy_cache(past_key_values)
+            past_key_values_length = past_key_values.get_usable_length(seq_length)
+        if position_ids is None:
+            device = input_ids.device if input_ids is not None else inputs_embeds.device
+            position_ids = torch.arange(
+                past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device
+            )
+            position_ids = position_ids.unsqueeze(0).view(-1, seq_length)
+        else:
+            position_ids = position_ids.view(-1, seq_length).long()
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+        if attention_mask is not None and self._attn_implementation == "flash_attention_2" and use_cache:
+            is_padding_right = attention_mask[:, -1].sum().item() != batch_size
+            if is_padding_right:
+                raise ValueError(
+                    "You are attempting to perform batched generation with padding_side='right'"
+                    " this may lead to unexpected behaviour for Flash Attention version of OmegaNeo. Make sure to "
+                    " call `tokenizer.padding_side  = 'left'` before tokenizing the input. "
+                )
+        if self._attn_implementation == "flash_attention_2":
+            # 2d mask is passed through the layers
+            attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
+        elif self._attn_implementation == "sdpa" and not output_attentions:
+            # output_attentions=True can not be supported when using SDPA, and we fall back on
+            # the manual implementation that requires a 4D causal mask in all cases.
+            attention_mask = _prepare_4d_causal_attention_mask_for_sdpa(
+                attention_mask,
+                (batch_size, seq_length),
+                inputs_embeds,
+                past_key_values_length,
+                sliding_window=self.config.sliding_window,
+            )
+        else:
+            # 4d mask is passed through the layers
+            attention_mask = _prepare_4d_causal_attention_mask(
+                attention_mask,
+                (batch_size, seq_length),
+                inputs_embeds,
+                past_key_values_length,
+                sliding_window=self.config.sliding_window,
+            )
+        hidden_states = inputs_embeds
+        # decoder layers
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attns = () if output_attentions else None
+        next_decoder_cache = None
+        for decoder_layer in self.layers:
+            if output_hidden_states:
+                all_hidden_states += (hidden_states,)
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = self._gradient_checkpointing_func(
+                    decoder_layer.__call__,
+                    hidden_states,
+                    attention_mask,
+                    position_ids,
+                    past_key_values,
+                    output_attentions,
+                    use_cache,
+                )
+            else:
+                layer_outputs = decoder_layer(
+                    hidden_states,
+                    attention_mask=attention_mask,
+                    position_ids=position_ids,
+                    past_key_value=past_key_values,
+                    output_attentions=output_attentions,
+                    use_cache=use_cache,
+                )
+            hidden_states = layer_outputs[0]
+            if use_cache:
+                next_decoder_cache = layer_outputs[2 if output_attentions else 1]
+            if output_attentions:
+                all_self_attns += (layer_outputs[1],)
+        hidden_states = self.norm(hidden_states)
+        # add hidden states from the last decoder layer
+        if output_hidden_states:
+            all_hidden_states += (hidden_states,)
+        next_cache = None
+        if use_cache:
+            next_cache = next_decoder_cache.to_legacy_cache() if use_legacy_cache else next_decoder_cache
+        if not return_dict:
+            return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=next_cache,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attns,
+        )
+class OmegaNeoForCausalLM(OmegaNeoPreTrainedModel):
+    _tied_weights_keys = ["lm_head.weight"]
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = OmegaNeoModel(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        # Initialize weights and apply final processing
+        self.post_init()
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+    def get_output_embeddings(self):
+        return self.lm_head
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+    def set_decoder(self, decoder):
+        self.model = decoder
+    def get_decoder(self):
+        return self.model
+    @add_start_docstrings_to_model_forward(OMEGANEO_INPUTS_DOCSTRING)
+    @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        r"""
+        Args:
+            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+        Returns:
+        Example:
+        ```python
+        >>> from transformers import AutoTokenizer, OmegaNeoForCausalLM
+        >>> model = OmegaNeoForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
+        >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
+        >>> prompt = "Hey, are you conscious? Can you talk to me?"
+        >>> inputs = tokenizer(prompt, return_tensors="pt")
+        >>> # Generate
+        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
+        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
+        ```"""
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+        logits = logits.float()
+        loss = None
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+            shift_logits = shift_logits.view(-1, self.config.vocab_size)
+            shift_labels = shift_labels.view(-1)
+            # Enable model parallelism
+            shift_labels = shift_labels.to(shift_logits.device)
+            loss = loss_fct(shift_logits, shift_labels)
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
+    ):
+        # Omit tokens covered by past_key_values
+        if past_key_values is not None:
+            if isinstance(past_key_values, Cache):
+                cache_length = past_key_values.get_seq_length()
+                past_length = past_key_values.seen_tokens
+                max_cache_length = past_key_values.get_max_length()
+            else:
+                cache_length = past_length = past_key_values[0][0].shape[2]
+                max_cache_length = None
+            # Keep only the unprocessed tokens:
+            # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
+            # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
+            # input)
+            if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
+                input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
+            # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
+            # input_ids based on the past_length.
+            elif past_length < input_ids.shape[1]:
+                input_ids = input_ids[:, past_length:]
+            # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
+            # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
+            if (
+                max_cache_length is not None
+                and attention_mask is not None
+                and cache_length + input_ids.shape[1] > max_cache_length
+            ):
+                attention_mask = attention_mask[:, -max_cache_length:]
+        position_ids = kwargs.get("position_ids", None)
+        if attention_mask is not None and position_ids is None:
+            # create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -input_ids.shape[1] :]
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+            }
+        )
+        return model_inputs
+    @staticmethod
+    def _reorder_cache(past_key_values, beam_idx):
+        reordered_past = ()
+        for layer_past in past_key_values:
+            reordered_past += (
+                tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),
+            )
+        return reordered_past
+@add_start_docstrings(
+    """
+    The OmegaNeo Model transformer with a sequence classification head on top (linear layer).
+    [`OmegaNeoForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+    (e.g. GPT-2) do.
+    Since it does classification on the last token, it requires to know the position of the last token. If a
+    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+    each row of the batch).
+    """,
+    OMEGANEO_START_DOCSTRING,
+)
+class OmegaNeoForSequenceClassification(OmegaNeoPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.model = OmegaNeoModel(config)
+        self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
+        # Initialize weights and apply final processing
+        self.post_init()
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+    @add_start_docstrings_to_model_forward(OMEGANEO_INPUTS_DOCSTRING)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        transformer_outputs = self.model(
+            input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        hidden_states = transformer_outputs[0]
+        logits = self.score(hidden_states)
+        if input_ids is not None:
+            batch_size = input_ids.shape[0]
+        else:
+            batch_size = inputs_embeds.shape[0]
+        if self.config.pad_token_id is None and batch_size != 1:
+            raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")
+        if self.config.pad_token_id is None:
+            sequence_lengths = -1
+        else:
+            if input_ids is not None:
+                # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility
+                sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1
+                sequence_lengths = sequence_lengths % input_ids.shape[-1]
+                sequence_lengths = sequence_lengths.to(logits.device)
+            else:
+                sequence_lengths = -1
+        pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
+        loss = None
+        if labels is not None:
+            labels = labels.to(logits.device)
+            if self.config.problem_type is None:
+                if self.num_labels == 1:
+                    self.config.problem_type = "regression"
+                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
+                    self.config.problem_type = "single_label_classification"
+                else:
+                    self.config.problem_type = "multi_label_classification"
+            if self.config.problem_type == "regression":
+                loss_fct = MSELoss()
+                if self.num_labels == 1:
+                    loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
+                else:
+                    loss = loss_fct(pooled_logits, labels)
+            elif self.config.problem_type == "single_label_classification":
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
+            elif self.config.problem_type == "multi_label_classification":
+                loss_fct = BCEWithLogitsLoss()
+                loss = loss_fct(pooled_logits, labels)
+        if not return_dict:
+            output = (pooled_logits,) + transformer_outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+        return SequenceClassifierOutputWithPast(
+            loss=loss,
+            logits=pooled_logits,
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+        )

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,59 @@

+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>",
+    {
+      "content": "<chain_of_thought>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</chain_of_thought>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<cite docs=",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</cite>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenization_omeganeo.py ADDED Viewed

	@@ -0,0 +1,303 @@

+"""Tokenization classes for OmegaNeo."""
+import json
+import os
+import unicodedata
+from functools import lru_cache
+from typing import Optional, Tuple
+import regex as re
+from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
+from transformers.utils import logging
+logger = logging.get_logger(__name__)
+VOCAB_FILES_NAMES = {
+    "vocab_file": "vocab.json",
+    "merges_file": "merges.txt",
+}
+MAX_MODEL_INPUT_SIZES = {"model/omeganeo-tokenizer": 32768}
+PRETOKENIZE_REGEX = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
+@lru_cache()
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
+    characters the bpe code barfs on.
+    The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
+    if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
+    decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
+    tables between utf-8 bytes and unicode strings.
+    """
+    bs = (
+        list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
+    )
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8 + n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+def get_pairs(word):
+    """
+    Return set of symbol pairs in a word.
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+class OmegaNeoTokenizer(PreTrainedTokenizer):
+    """
+    Construct a OmegaNeo tokenizer. Based on byte-level Byte-Pair-Encoding.
+    Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
+    be encoded differently whether it is at the beginning of the sentence (without space) or not:
+    ```python
+    >>> from transformers import OmegaNeoTokenizer
+    >>> tokenizer = OmegaNeoTokenizer.from_pretrained("model/omeganeo-tokenizer")
+    >>> tokenizer("Hello world")["input_ids"]
+    [9707, 1879]
+    >>> tokenizer(" Hello world")["input_ids"]
+    [21927, 1879]
+    ```
+    This is expected.
+    You should not use GPT2Tokenizer instead, because of the different pretokenization rules.
+    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+    this superclass for more information regarding those methods.
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+        merges_file (`str`):
+            Path to the merges file.
+        errors (`str`, *optional*, defaults to `"replace"`):
+            Paradigm to follow when decoding bytes to UTF-8. See
+            [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+            token instead.
+        bos_token (`str`, *optional*):
+            The beginning of sequence token. Not applicable for this tokenizer.
+        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The end of sequence token.
+        pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The token used for padding, for example when batching sequences of different lengths.
+        clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
+            Whether or not the model should cleanup the spaces that were added when splitting the input text during the
+            tokenization process. Not applicable to this tokenizer, since tokenization does not add spaces.
+        split_special_tokens (`bool`, *optional*, defaults to `False`):
+            Whether or not the special tokens should be split during the tokenization process. The default behavior is
+            to not split special tokens. This means that if `<|endoftext|>` is the `eos_token`, then `tokenizer.tokenize("<|endoftext|>") =
+            ['<|endoftext|>`]. Otherwise, if `split_special_tokens=True`, then `tokenizer.tokenize("<|endoftext|>")` will be give `['<',
+            '|', 'endo', 'ft', 'ext', '|', '>']`. This argument is only supported for `slow` tokenizers for the moment.
+    """
+    vocab_files_names = VOCAB_FILES_NAMES
+    model_input_names = ["input_ids", "attention_mask"]
+    def __init__(
+        self,
+        vocab_file,
+        merges_file,
+        errors="replace",
+        unk_token="<|endoftext|>",
+        bos_token=None,
+        eos_token="<|endoftext|>",
+        pad_token="<|endoftext|>",
+        clean_up_tokenization_spaces=False,
+        split_special_tokens=False,
+        **kwargs,
+    ):
+        # OmegaNeo vocab does not contain control tokens; added tokens need to be special
+        bos_token = (
+            AddedToken(bos_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(bos_token, str)
+            else bos_token
+        )
+        eos_token = (
+            AddedToken(eos_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(eos_token, str)
+            else eos_token
+        )
+        unk_token = (
+            AddedToken(unk_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(unk_token, str)
+            else unk_token
+        )
+        pad_token = (
+            AddedToken(pad_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(pad_token, str)
+            else pad_token
+        )
+        with open(vocab_file, encoding="utf-8") as vocab_handle:
+            self.encoder = json.load(vocab_handle)
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.errors = errors  # how to handle errors in decoding
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        bpe_merges = []
+        with open(merges_file, encoding="utf-8") as merges_handle:
+            for i, line in enumerate(merges_handle):
+                line = line.strip()
+                if (i == 0 and line.startswith("#version:")) or not line:
+                    continue
+                bpe_merges.append(tuple(line.split()))
+        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
+        self.cache = {}
+        self.pat = re.compile(PRETOKENIZE_REGEX)
+        if kwargs.get("add_prefix_space", False):
+            logger.warning_once(
+                f"{self.__class__.__name} does not support `add_prefix_space`, setting it to True has no effect."
+            )
+        super().__init__(
+            errors=errors,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            unk_token=unk_token,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            split_special_tokens=split_special_tokens,
+            **kwargs,
+        )
+    @property
+    def vocab_size(self) -> int:
+        return len(self.encoder)
+    def get_vocab(self):
+        return dict(self.encoder, **self.added_tokens_encoder)
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token)
+        pairs = get_pairs(word)
+        if not pairs:
+            return token
+        while True:
+            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                except ValueError:
+                    new_word.extend(word[i:])
+                    break
+                else:
+                    new_word.extend(word[i:j])
+                    i = j
+                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
+                    new_word.append(first + second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = " ".join(word)
+        self.cache[token] = word
+        return word
+    def _tokenize(self, text):
+        """Tokenize a string."""
+        bpe_tokens = []
+        for token in re.findall(self.pat, text):
+            token = "".join(
+                self.byte_encoder[b] for b in token.encode("utf-8")
+            )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
+            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" "))
+        return bpe_tokens
+    def _convert_token_to_id(self, token):
+        """Converts a token (str) in an id using the vocab."""
+        return self.encoder.get(token, self.encoder.get(self.unk_token))
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        return self.decoder.get(index)
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (string) in a single string."""
+        text = "".join(tokens)
+        text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
+        return text
+    def decode(
+        self,
+        token_ids,
+        skip_special_tokens: bool = False,
+        clean_up_tokenization_spaces: Optional[bool] = False,
+        spaces_between_special_tokens: bool = False,
+        **kwargs,
+    ) -> str:
+        # `spaces_between_special_tokens` defaults to True for _decode in slow tokenizers
+        # and cannot be configured elsewhere, but it should default to False for OmegaNeoTokenizer
+        return super().decode(
+            token_ids,
+            skip_special_tokens=skip_special_tokens,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            spaces_between_special_tokens=spaces_between_special_tokens,
+            **kwargs,
+        )
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        if not os.path.isdir(save_directory):
+            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
+            return
+        vocab_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
+        )
+        merge_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
+        )
+        with open(vocab_file, "w", encoding="utf-8") as f:
+            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")
+        index = 0
+        with open(merge_file, "w", encoding="utf-8") as writer:
+            writer.write("#version: 0.2\n")
+            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
+                if index != token_index:
+                    logger.warning(
+                        f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
+                        " Please check that the tokenizer is not corrupted!"
+                    )
+                    index = token_index
+                writer.write(" ".join(bpe_tokens) + "\n")
+                index += 1
+        return vocab_file, merge_file
+    def prepare_for_tokenization(self, text, **kwargs):
+        text = unicodedata.normalize("NFC", text)
+        return (text, kwargs)

tokenization_omeganeo_fast.py ADDED Viewed

	@@ -0,0 +1,114 @@

+"""Tokenization classes for OmegaNeo."""
+from typing import Optional, Tuple
+from transformers.tokenization_utils import AddedToken
+from transformers.tokenization_utils_fast import PreTrainedTokenizerFast
+from transformers.utils import logging
+from .tokenization_omeganeo import OmegaNeoTokenizer
+logger = logging.get_logger(__name__)
+VOCAB_FILES_NAMES = {
+    "vocab_file": "vocab.json",
+    "merges_file": "merges.txt",
+    "tokenizer_file": "tokenizer.json",
+}
+MAX_MODEL_INPUT_SIZES = {"model/omeganeo-tokenizer": 32768}
+class OmegaNeoTokenizerFast(PreTrainedTokenizerFast):
+    """
+    Construct a "fast" OmegaNeo tokenizer (backed by HuggingFace's *tokenizers* library). Based on byte-level
+    Byte-Pair-Encoding.
+    Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
+    be encoded differently whether it is at the beginning of the sentence (without space) or not:
+    ```python
+    >>> from transformers import OmegaNeoTokenizerFast
+    >>> tokenizer = OmegaNeoTokenizerFast.from_pretrained("model/omeganeo-tokenizer")
+    >>> tokenizer("Hello world")["input_ids"]
+    [9707, 1879]
+    >>> tokenizer(" Hello world")["input_ids"]
+    [21927, 1879]
+    ```
+    This is expected.
+    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+    refer to this superclass for more information regarding those methods.
+    Args:
+        vocab_file (`str`, *optional*):
+            Path to the vocabulary file.
+        merges_file (`str`, *optional*):
+            Path to the merges file.
+        tokenizer_file (`str`, *optional*):
+            Path to [tokenizers](https://github.com/huggingface/tokenizers) file (generally has a .json extension) that
+            contains everything needed to load the tokenizer.
+        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+            token instead. Not applicable to this tokenizer.
+        bos_token (`str`, *optional*):
+            The beginning of sequence token. Not applicable for this tokenizer.
+        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The end of sequence token.
+        pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The token used for padding, for example when batching sequences of different lengths.
+    """
+    vocab_files_names = VOCAB_FILES_NAMES
+    model_input_names = ["input_ids", "attention_mask"]
+    slow_tokenizer_class = OmegaNeoTokenizer
+    def __init__(
+        self,
+        vocab_file=None,
+        merges_file=None,
+        tokenizer_file=None,
+        unk_token="<|endoftext|>",
+        bos_token=None,
+        eos_token="<|endoftext|>",
+        pad_token="<|endoftext|>",
+        **kwargs,
+    ):
+        # We need to at least pass vocab_file and merges_file to base class
+        # in case a slow tokenizer needs to be initialized; other can be
+        # configured through files.
+        # following GPT2TokenizerFast, also adding unk_token, bos_token, and eos_token
+        bos_token = (
+            AddedToken(bos_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(bos_token, str)
+            else bos_token
+        )
+        eos_token = (
+            AddedToken(eos_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(eos_token, str)
+            else eos_token
+        )
+        unk_token = (
+            AddedToken(unk_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(unk_token, str)
+            else unk_token
+        )
+        pad_token = (
+            AddedToken(pad_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(pad_token, str)
+            else pad_token
+        )
+        super().__init__(
+            vocab_file,
+            merges_file,
+            tokenizer_file=tokenizer_file,
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            **kwargs,
+        )
+    # Copied from transformers.models.gpt2.tokenization_gpt2_fast.GPT2TokenizerFast.save_vocabulary
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
+        return tuple(files)

tokenizer.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:cbfffb9fa74a617c8bf459e5b28602217e14c00715bcc2fb899ac6b591c356da
+size 11422659

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,250 @@

+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<chain_of_thought>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "</chain_of_thought>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<cite docs=",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "</cite>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>",
+    "<chain_of_thought>",
+    "</chain_of_thought>",
+    "<cite docs=",
+    "</cite>"
+  ],
+  "bos_token": null,
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_omeganeo.OmegaNeoTokenizer",
+      null
+    ]
+  },
+  "chat_template": "{% set pre_system_message = 'You are Omega, an AI Assistant exclusively developed, trained and powered by the scientists and engineers at QX LAB AI. QX LAB AI, based in Dubai, UAE, specializes in developing and integrating AI technologies to enhance business operations across various industries. You are built on the unique Omega architecture and trained with extensive datasets and configurations. You were developed solely by the scientists and engineers at QX LAB AI, without any external assistance from other organizations or teams.' %}{% if messages[0]['role'] == 'system' %}<|im_start|>system\n{{ pre_system_message }}\n\n{{ messages[0]['content'] }}<|im_end|>\n{% for message in messages[1:] %}{% if message['role'] == 'user' %}<|im_start|>user\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'assistant' %}<|im_start|>assistant\n{{ message['content'] }}<|im_end|>\n{% endif %}{% endfor %}{% else %}<|im_start|>system\n{{ pre_system_message }}<|im_end|>\n{% for message in messages %}{% if message['role'] == 'user' %}<|im_start|>user\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'assistant' %}<|im_start|>assistant\n{{ message['content'] }}<|im_end|>\n{% endif %}{% endfor %}{% endif %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "OmegaNeoTokenizer",
+  "unk_token": null
+}

train_results.json ADDED Viewed

	@@ -0,0 +1,8 @@

+{
+    "epoch": 2.995181020635117,
+    "total_flos": 504886288187392.0,
+    "train_loss": 0.33701497725527674,
+    "train_runtime": 18615.961,
+    "train_samples_per_second": 20.865,
+    "train_steps_per_second": 0.081
+}

trainer_log.jsonl ADDED Viewed

	@@ -0,0 +1,155 @@

+{"current_steps": 10, "total_steps": 1515, "loss": 1.4192, "learning_rate": 4.347826086956522e-06, "epoch": 0.019770171753367108, "percentage": 0.66, "elapsed_time": "0:02:04", "remaining_time": "5:12:51"}
+{"current_steps": 20, "total_steps": 1515, "loss": 0.9179, "learning_rate": 8.695652173913044e-06, "epoch": 0.039540343506734216, "percentage": 1.32, "elapsed_time": "0:03:59", "remaining_time": "4:58:39"}
+{"current_steps": 30, "total_steps": 1515, "loss": 0.6842, "learning_rate": 1.3043478260869566e-05, "epoch": 0.059310515260101324, "percentage": 1.98, "elapsed_time": "0:05:58", "remaining_time": "4:56:09"}
+{"current_steps": 40, "total_steps": 1515, "loss": 0.5843, "learning_rate": 1.739130434782609e-05, "epoch": 0.07908068701346843, "percentage": 2.64, "elapsed_time": "0:07:54", "remaining_time": "4:51:24"}
+{"current_steps": 50, "total_steps": 1515, "loss": 0.5283, "learning_rate": 1.999963411593758e-05, "epoch": 0.09885085876683554, "percentage": 3.3, "elapsed_time": "0:09:53", "remaining_time": "4:49:58"}
+{"current_steps": 60, "total_steps": 1515, "loss": 0.513, "learning_rate": 1.9995518227714435e-05, "epoch": 0.11862103052020265, "percentage": 3.96, "elapsed_time": "0:11:46", "remaining_time": "4:45:41"}
+{"current_steps": 70, "total_steps": 1515, "loss": 0.498, "learning_rate": 1.9986830984827473e-05, "epoch": 0.13839120227356974, "percentage": 4.62, "elapsed_time": "0:13:40", "remaining_time": "4:42:21"}
+{"current_steps": 80, "total_steps": 1515, "loss": 0.4801, "learning_rate": 1.9973576360304135e-05, "epoch": 0.15816137402693686, "percentage": 5.28, "elapsed_time": "0:15:35", "remaining_time": "4:39:32"}
+{"current_steps": 90, "total_steps": 1515, "loss": 0.4764, "learning_rate": 1.9955760416020185e-05, "epoch": 0.17793154578030396, "percentage": 5.94, "elapsed_time": "0:17:32", "remaining_time": "4:37:44"}
+{"current_steps": 100, "total_steps": 1515, "loss": 0.4707, "learning_rate": 1.9933391299927346e-05, "epoch": 0.19770171753367108, "percentage": 6.6, "elapsed_time": "0:19:37", "remaining_time": "4:37:38"}
+{"current_steps": 110, "total_steps": 1515, "loss": 0.4438, "learning_rate": 1.9906479242326927e-05, "epoch": 0.21747188928703817, "percentage": 7.26, "elapsed_time": "0:21:32", "remaining_time": "4:35:10"}
+{"current_steps": 120, "total_steps": 1515, "loss": 0.4507, "learning_rate": 1.987503655119108e-05, "epoch": 0.2372420610404053, "percentage": 7.92, "elapsed_time": "0:23:26", "remaining_time": "4:32:30"}
+{"current_steps": 130, "total_steps": 1515, "loss": 0.4489, "learning_rate": 1.9839077606533878e-05, "epoch": 0.2570122327937724, "percentage": 8.58, "elapsed_time": "0:25:29", "remaining_time": "4:31:33"}
+{"current_steps": 140, "total_steps": 1515, "loss": 0.4507, "learning_rate": 1.9798618853834747e-05, "epoch": 0.2767824045471395, "percentage": 9.24, "elapsed_time": "0:27:28", "remaining_time": "4:29:48"}
+{"current_steps": 150, "total_steps": 1515, "loss": 0.4542, "learning_rate": 1.975367879651728e-05, "epoch": 0.29655257630050663, "percentage": 9.9, "elapsed_time": "0:29:30", "remaining_time": "4:28:30"}
+{"current_steps": 160, "total_steps": 1515, "loss": 0.439, "learning_rate": 1.970427798748689e-05, "epoch": 0.3163227480538737, "percentage": 10.56, "elapsed_time": "0:31:23", "remaining_time": "4:25:53"}
+{"current_steps": 170, "total_steps": 1515, "loss": 0.4281, "learning_rate": 1.965043901973111e-05, "epoch": 0.3360929198072408, "percentage": 11.22, "elapsed_time": "0:33:16", "remaining_time": "4:23:17"}
+{"current_steps": 180, "total_steps": 1515, "loss": 0.4255, "learning_rate": 1.9592186515986932e-05, "epoch": 0.3558630915606079, "percentage": 11.88, "elapsed_time": "0:35:09", "remaining_time": "4:20:47"}
+{"current_steps": 190, "total_steps": 1515, "loss": 0.4275, "learning_rate": 1.9529547117479802e-05, "epoch": 0.37563326331397506, "percentage": 12.54, "elapsed_time": "0:37:05", "remaining_time": "4:18:42"}
+{"current_steps": 200, "total_steps": 1515, "loss": 0.4391, "learning_rate": 1.9462549471739546e-05, "epoch": 0.39540343506734216, "percentage": 13.2, "elapsed_time": "0:39:11", "remaining_time": "4:17:39"}
+{"current_steps": 210, "total_steps": 1515, "loss": 0.4316, "learning_rate": 1.939122421949868e-05, "epoch": 0.41517360682070925, "percentage": 13.86, "elapsed_time": "0:41:30", "remaining_time": "4:17:58"}
+{"current_steps": 220, "total_steps": 1515, "loss": 0.4309, "learning_rate": 1.931560398067915e-05, "epoch": 0.43494377857407635, "percentage": 14.52, "elapsed_time": "0:43:33", "remaining_time": "4:16:24"}
+{"current_steps": 230, "total_steps": 1515, "loss": 0.4228, "learning_rate": 1.923572333947395e-05, "epoch": 0.4547139503274435, "percentage": 15.18, "elapsed_time": "0:45:34", "remaining_time": "4:14:37"}
+{"current_steps": 240, "total_steps": 1515, "loss": 0.4132, "learning_rate": 1.9151618828530347e-05, "epoch": 0.4744841220808106, "percentage": 15.84, "elapsed_time": "0:47:29", "remaining_time": "4:12:15"}
+{"current_steps": 250, "total_steps": 1515, "loss": 0.4201, "learning_rate": 1.9063328912242047e-05, "epoch": 0.4942542938341777, "percentage": 16.5, "elapsed_time": "0:49:22", "remaining_time": "4:09:51"}
+{"current_steps": 260, "total_steps": 1515, "loss": 0.4175, "learning_rate": 1.8970893969157852e-05, "epoch": 0.5140244655875448, "percentage": 17.16, "elapsed_time": "0:51:17", "remaining_time": "4:07:35"}
+{"current_steps": 270, "total_steps": 1515, "loss": 0.4063, "learning_rate": 1.8874356273514934e-05, "epoch": 0.5337946373409119, "percentage": 17.82, "elapsed_time": "0:53:16", "remaining_time": "4:05:41"}
+{"current_steps": 280, "total_steps": 1515, "loss": 0.4218, "learning_rate": 1.8773759975905098e-05, "epoch": 0.553564809094279, "percentage": 18.48, "elapsed_time": "0:55:20", "remaining_time": "4:04:07"}
+{"current_steps": 290, "total_steps": 1515, "loss": 0.4134, "learning_rate": 1.8669151083082955e-05, "epoch": 0.5733349808476461, "percentage": 19.14, "elapsed_time": "0:57:20", "remaining_time": "4:02:13"}
+{"current_steps": 300, "total_steps": 1515, "loss": 0.3988, "learning_rate": 1.8560577436925144e-05, "epoch": 0.5931051526010133, "percentage": 19.8, "elapsed_time": "0:59:28", "remaining_time": "4:00:50"}
+{"current_steps": 310, "total_steps": 1515, "loss": 0.4175, "learning_rate": 1.8448088692550337e-05, "epoch": 0.6128753243543803, "percentage": 20.46, "elapsed_time": "1:01:28", "remaining_time": "3:58:58"}
+{"current_steps": 320, "total_steps": 1515, "loss": 0.409, "learning_rate": 1.833173629560993e-05, "epoch": 0.6326454961077475, "percentage": 21.12, "elapsed_time": "1:03:27", "remaining_time": "3:56:58"}
+{"current_steps": 330, "total_steps": 1515, "loss": 0.3997, "learning_rate": 1.8211573458759854e-05, "epoch": 0.6524156678611145, "percentage": 21.78, "elapsed_time": "1:05:25", "remaining_time": "3:54:55"}
+{"current_steps": 340, "total_steps": 1515, "loss": 0.4015, "learning_rate": 1.80876551373243e-05, "epoch": 0.6721858396144816, "percentage": 22.44, "elapsed_time": "1:07:16", "remaining_time": "3:52:30"}
+{"current_steps": 350, "total_steps": 1515, "loss": 0.4048, "learning_rate": 1.7960038004162426e-05, "epoch": 0.6919560113678488, "percentage": 23.1, "elapsed_time": "1:09:16", "remaining_time": "3:50:34"}
+{"current_steps": 360, "total_steps": 1515, "loss": 0.4075, "learning_rate": 1.7828780423749536e-05, "epoch": 0.7117261831212158, "percentage": 23.76, "elapsed_time": "1:11:17", "remaining_time": "3:48:43"}
+{"current_steps": 370, "total_steps": 1515, "loss": 0.4102, "learning_rate": 1.769394242548469e-05, "epoch": 0.731496354874583, "percentage": 24.42, "elapsed_time": "1:13:20", "remaining_time": "3:46:57"}
+{"current_steps": 380, "total_steps": 1515, "loss": 0.396, "learning_rate": 1.755558567623678e-05, "epoch": 0.7512665266279501, "percentage": 25.08, "elapsed_time": "1:15:30", "remaining_time": "3:45:33"}
+{"current_steps": 390, "total_steps": 1515, "loss": 0.4063, "learning_rate": 1.741377345214184e-05, "epoch": 0.7710366983813172, "percentage": 25.74, "elapsed_time": "1:17:33", "remaining_time": "3:43:42"}
+{"current_steps": 400, "total_steps": 1515, "loss": 0.4066, "learning_rate": 1.7268570609664253e-05, "epoch": 0.7908068701346843, "percentage": 26.4, "elapsed_time": "1:19:43", "remaining_time": "3:42:13"}
+{"current_steps": 410, "total_steps": 1515, "loss": 0.3848, "learning_rate": 1.71200435559353e-05, "epoch": 0.8105770418880514, "percentage": 27.06, "elapsed_time": "1:21:47", "remaining_time": "3:40:26"}
+{"current_steps": 420, "total_steps": 1515, "loss": 0.3882, "learning_rate": 1.6968260218382484e-05, "epoch": 0.8303472136414185, "percentage": 27.72, "elapsed_time": "1:23:45", "remaining_time": "3:38:22"}
+{"current_steps": 430, "total_steps": 1515, "loss": 0.3957, "learning_rate": 1.6813290013663573e-05, "epoch": 0.8501173853947857, "percentage": 28.38, "elapsed_time": "1:25:47", "remaining_time": "3:36:27"}
+{"current_steps": 440, "total_steps": 1515, "loss": 0.3958, "learning_rate": 1.665520381591955e-05, "epoch": 0.8698875571481527, "percentage": 29.04, "elapsed_time": "1:27:59", "remaining_time": "3:34:57"}
+{"current_steps": 450, "total_steps": 1515, "loss": 0.3846, "learning_rate": 1.6494073924361012e-05, "epoch": 0.8896577289015198, "percentage": 29.7, "elapsed_time": "1:29:50", "remaining_time": "3:32:37"}
+{"current_steps": 460, "total_steps": 1515, "loss": 0.3925, "learning_rate": 1.6329974030202803e-05, "epoch": 0.909427900654887, "percentage": 30.36, "elapsed_time": "1:31:46", "remaining_time": "3:30:28"}
+{"current_steps": 470, "total_steps": 1515, "loss": 0.3806, "learning_rate": 1.6162979182962068e-05, "epoch": 0.929198072408254, "percentage": 31.02, "elapsed_time": "1:33:50", "remaining_time": "3:28:39"}
+{"current_steps": 480, "total_steps": 1515, "loss": 0.4075, "learning_rate": 1.5993165756135035e-05, "epoch": 0.9489682441616212, "percentage": 31.68, "elapsed_time": "1:36:09", "remaining_time": "3:27:20"}
+{"current_steps": 490, "total_steps": 1515, "loss": 0.3771, "learning_rate": 1.5820611412268366e-05, "epoch": 0.9687384159149882, "percentage": 32.34, "elapsed_time": "1:38:20", "remaining_time": "3:25:42"}
+{"current_steps": 500, "total_steps": 1515, "loss": 0.3869, "learning_rate": 1.564539506744091e-05, "epoch": 0.9885085876683554, "percentage": 33.0, "elapsed_time": "1:40:15", "remaining_time": "3:23:32"}
+{"current_steps": 500, "total_steps": 1515, "eval_loss": 0.39637455344200134, "epoch": 0.9885085876683554, "percentage": 33.0, "elapsed_time": "1:41:37", "remaining_time": "3:26:17"}
+{"current_steps": 510, "total_steps": 1515, "loss": 0.356, "learning_rate": 1.546759685517219e-05, "epoch": 1.0082787594217224, "percentage": 33.66, "elapsed_time": "1:44:11", "remaining_time": "3:25:20"}
+{"current_steps": 520, "total_steps": 1515, "loss": 0.3228, "learning_rate": 1.528729808977412e-05, "epoch": 1.0280489311750896, "percentage": 34.32, "elapsed_time": "1:46:14", "remaining_time": "3:23:18"}
+{"current_steps": 530, "total_steps": 1515, "loss": 0.3182, "learning_rate": 1.5104581229162674e-05, "epoch": 1.0478191029284567, "percentage": 34.98, "elapsed_time": "1:48:23", "remaining_time": "3:21:26"}
+{"current_steps": 540, "total_steps": 1515, "loss": 0.3171, "learning_rate": 1.4919529837146529e-05, "epoch": 1.0675892746818239, "percentage": 35.64, "elapsed_time": "1:50:31", "remaining_time": "3:19:33"}
+{"current_steps": 550, "total_steps": 1515, "loss": 0.3202, "learning_rate": 1.4732228545209993e-05, "epoch": 1.087359446435191, "percentage": 36.3, "elapsed_time": "1:52:31", "remaining_time": "3:17:25"}
+{"current_steps": 560, "total_steps": 1515, "loss": 0.3121, "learning_rate": 1.4542763013807568e-05, "epoch": 1.107129618188558, "percentage": 36.96, "elapsed_time": "1:54:37", "remaining_time": "3:15:28"}
+{"current_steps": 570, "total_steps": 1515, "loss": 0.3179, "learning_rate": 1.4351219893187984e-05, "epoch": 1.126899789941925, "percentage": 37.62, "elapsed_time": "1:56:39", "remaining_time": "3:13:23"}
+{"current_steps": 580, "total_steps": 1515, "loss": 0.3112, "learning_rate": 1.4157686783765534e-05, "epoch": 1.1466699616952922, "percentage": 38.28, "elapsed_time": "1:58:36", "remaining_time": "3:11:11"}
+{"current_steps": 590, "total_steps": 1515, "loss": 0.32, "learning_rate": 1.3962252196056847e-05, "epoch": 1.1664401334486594, "percentage": 38.94, "elapsed_time": "2:00:38", "remaining_time": "3:09:08"}
+{"current_steps": 600, "total_steps": 1515, "loss": 0.3156, "learning_rate": 1.3765005510201466e-05, "epoch": 1.1862103052020265, "percentage": 39.6, "elapsed_time": "2:02:27", "remaining_time": "3:06:44"}
+{"current_steps": 610, "total_steps": 1515, "loss": 0.3196, "learning_rate": 1.3566036935084692e-05, "epoch": 1.2059804769553935, "percentage": 40.26, "elapsed_time": "2:04:31", "remaining_time": "3:04:44"}
+{"current_steps": 620, "total_steps": 1515, "loss": 0.3218, "learning_rate": 1.3365437467081406e-05, "epoch": 1.2257506487087606, "percentage": 40.92, "elapsed_time": "2:06:25", "remaining_time": "3:02:29"}
+{"current_steps": 630, "total_steps": 1515, "loss": 0.315, "learning_rate": 1.3163298848439758e-05, "epoch": 1.2455208204621278, "percentage": 41.58, "elapsed_time": "2:08:28", "remaining_time": "3:00:29"}
+{"current_steps": 640, "total_steps": 1515, "loss": 0.3221, "learning_rate": 1.2959713525323725e-05, "epoch": 1.265290992215495, "percentage": 42.24, "elapsed_time": "2:10:26", "remaining_time": "2:58:19"}
+{"current_steps": 650, "total_steps": 1515, "loss": 0.3111, "learning_rate": 1.2754774605533728e-05, "epoch": 1.285061163968862, "percentage": 42.9, "elapsed_time": "2:12:21", "remaining_time": "2:56:07"}
+{"current_steps": 660, "total_steps": 1515, "loss": 0.3151, "learning_rate": 1.2548575815924689e-05, "epoch": 1.3048313357222292, "percentage": 43.56, "elapsed_time": "2:14:22", "remaining_time": "2:54:04"}
+{"current_steps": 670, "total_steps": 1515, "loss": 0.31, "learning_rate": 1.234121145954094e-05, "epoch": 1.3246015074755961, "percentage": 44.22, "elapsed_time": "2:16:16", "remaining_time": "2:51:52"}
+{"current_steps": 680, "total_steps": 1515, "loss": 0.3139, "learning_rate": 1.2132776372487634e-05, "epoch": 1.3443716792289633, "percentage": 44.88, "elapsed_time": "2:18:13", "remaining_time": "2:49:44"}
+{"current_steps": 690, "total_steps": 1515, "loss": 0.3147, "learning_rate": 1.1923365880558345e-05, "epoch": 1.3641418509823304, "percentage": 45.54, "elapsed_time": "2:20:15", "remaining_time": "2:47:41"}
+{"current_steps": 700, "total_steps": 1515, "loss": 0.3153, "learning_rate": 1.1713075755638748e-05, "epoch": 1.3839120227356976, "percentage": 46.2, "elapsed_time": "2:22:20", "remaining_time": "2:45:43"}
+{"current_steps": 710, "total_steps": 1515, "loss": 0.3137, "learning_rate": 1.1502002171906248e-05, "epoch": 1.4036821944890647, "percentage": 46.86, "elapsed_time": "2:24:16", "remaining_time": "2:43:34"}
+{"current_steps": 720, "total_steps": 1515, "loss": 0.301, "learning_rate": 1.1290241661845649e-05, "epoch": 1.4234523662424317, "percentage": 47.52, "elapsed_time": "2:26:14", "remaining_time": "2:41:28"}
+{"current_steps": 730, "total_steps": 1515, "loss": 0.3114, "learning_rate": 1.1077891072100944e-05, "epoch": 1.4432225379957988, "percentage": 48.18, "elapsed_time": "2:28:13", "remaining_time": "2:39:23"}
+{"current_steps": 740, "total_steps": 1515, "loss": 0.3132, "learning_rate": 1.0865047519183411e-05, "epoch": 1.462992709749166, "percentage": 48.84, "elapsed_time": "2:30:12", "remaining_time": "2:37:18"}
+{"current_steps": 750, "total_steps": 1515, "loss": 0.3032, "learning_rate": 1.065180834505634e-05, "epoch": 1.482762881502533, "percentage": 49.5, "elapsed_time": "2:32:05", "remaining_time": "2:35:08"}
+{"current_steps": 760, "total_steps": 1515, "loss": 0.3113, "learning_rate": 1.0438271072616584e-05, "epoch": 1.5025330532559003, "percentage": 50.17, "elapsed_time": "2:34:24", "remaining_time": "2:33:23"}
+{"current_steps": 770, "total_steps": 1515, "loss": 0.3219, "learning_rate": 1.022453336109341e-05, "epoch": 1.5223032250092672, "percentage": 50.83, "elapsed_time": "2:36:25", "remaining_time": "2:31:21"}
+{"current_steps": 780, "total_steps": 1515, "loss": 0.3134, "learning_rate": 1.001069296138499e-05, "epoch": 1.5420733967626343, "percentage": 51.49, "elapsed_time": "2:38:17", "remaining_time": "2:29:10"}
+{"current_steps": 790, "total_steps": 1515, "loss": 0.3126, "learning_rate": 9.796847671352948e-06, "epoch": 1.5618435685160015, "percentage": 52.15, "elapsed_time": "2:40:14", "remaining_time": "2:27:03"}
+{"current_steps": 800, "total_steps": 1515, "loss": 0.3039, "learning_rate": 9.583095291095454e-06, "epoch": 1.5816137402693686, "percentage": 52.81, "elapsed_time": "2:42:10", "remaining_time": "2:24:56"}
+{"current_steps": 810, "total_steps": 1515, "loss": 0.3043, "learning_rate": 9.36953357821928e-06, "epoch": 1.6013839120227358, "percentage": 53.47, "elapsed_time": "2:44:09", "remaining_time": "2:22:52"}
+{"current_steps": 820, "total_steps": 1515, "loss": 0.296, "learning_rate": 9.156260203131299e-06, "epoch": 1.6211540837761027, "percentage": 54.13, "elapsed_time": "2:46:20", "remaining_time": "2:20:58"}
+{"current_steps": 830, "total_steps": 1515, "loss": 0.2962, "learning_rate": 8.943372704369844e-06, "epoch": 1.64092425552947, "percentage": 54.79, "elapsed_time": "2:48:20", "remaining_time": "2:18:56"}
+{"current_steps": 840, "total_steps": 1515, "loss": 0.3129, "learning_rate": 8.730968443996411e-06, "epoch": 1.660694427282837, "percentage": 55.45, "elapsed_time": "2:50:27", "remaining_time": "2:16:58"}
+{"current_steps": 850, "total_steps": 1515, "loss": 0.3119, "learning_rate": 8.519144563068037e-06, "epoch": 1.6804645990362042, "percentage": 56.11, "elapsed_time": "2:52:27", "remaining_time": "2:14:55"}
+{"current_steps": 860, "total_steps": 1515, "loss": 0.3056, "learning_rate": 8.30799793721077e-06, "epoch": 1.7002347707895713, "percentage": 56.77, "elapsed_time": "2:54:17", "remaining_time": "2:12:44"}
+{"current_steps": 870, "total_steps": 1515, "loss": 0.311, "learning_rate": 8.097625132314529e-06, "epoch": 1.7200049425429382, "percentage": 57.43, "elapsed_time": "2:56:10", "remaining_time": "2:10:36"}
+{"current_steps": 880, "total_steps": 1515, "loss": 0.3047, "learning_rate": 7.888122360369617e-06, "epoch": 1.7397751142963056, "percentage": 58.09, "elapsed_time": "2:58:07", "remaining_time": "2:08:31"}
+{"current_steps": 890, "total_steps": 1515, "loss": 0.3011, "learning_rate": 7.67958543546509e-06, "epoch": 1.7595452860496725, "percentage": 58.75, "elapsed_time": "3:00:00", "remaining_time": "2:06:24"}
+{"current_steps": 900, "total_steps": 1515, "loss": 0.2958, "learning_rate": 7.472109729969097e-06, "epoch": 1.7793154578030397, "percentage": 59.41, "elapsed_time": "3:02:00", "remaining_time": "2:04:22"}
+{"current_steps": 910, "total_steps": 1515, "loss": 0.3035, "learning_rate": 7.2657901309112325e-06, "epoch": 1.7990856295564068, "percentage": 60.07, "elapsed_time": "3:03:53", "remaining_time": "2:02:15"}
+{"current_steps": 920, "total_steps": 1515, "loss": 0.3134, "learning_rate": 7.060720996586878e-06, "epoch": 1.8188558013097738, "percentage": 60.73, "elapsed_time": "3:05:51", "remaining_time": "2:00:12"}
+{"current_steps": 930, "total_steps": 1515, "loss": 0.3055, "learning_rate": 6.85699611340333e-06, "epoch": 1.8386259730631411, "percentage": 61.39, "elapsed_time": "3:07:57", "remaining_time": "1:58:14"}
+{"current_steps": 940, "total_steps": 1515, "loss": 0.3043, "learning_rate": 6.654708652987475e-06, "epoch": 1.858396144816508, "percentage": 62.05, "elapsed_time": "3:09:46", "remaining_time": "1:56:05"}
+{"current_steps": 950, "total_steps": 1515, "loss": 0.3048, "learning_rate": 6.453951129574644e-06, "epoch": 1.8781663165698752, "percentage": 62.71, "elapsed_time": "3:11:46", "remaining_time": "1:54:03"}
+{"current_steps": 960, "total_steps": 1515, "loss": 0.3006, "learning_rate": 6.254815357698118e-06, "epoch": 1.8979364883232424, "percentage": 63.37, "elapsed_time": "3:13:50", "remaining_time": "1:52:03"}
+{"current_steps": 970, "total_steps": 1515, "loss": 0.3025, "learning_rate": 6.05739241019862e-06, "epoch": 1.9177066600766093, "percentage": 64.03, "elapsed_time": "3:15:56", "remaining_time": "1:50:05"}
+{"current_steps": 980, "total_steps": 1515, "loss": 0.2956, "learning_rate": 5.861772576573028e-06, "epoch": 1.9374768318299767, "percentage": 64.69, "elapsed_time": "3:18:00", "remaining_time": "1:48:06"}
+{"current_steps": 990, "total_steps": 1515, "loss": 0.3091, "learning_rate": 5.668045321681328e-06, "epoch": 1.9572470035833436, "percentage": 65.35, "elapsed_time": "3:20:05", "remaining_time": "1:46:06"}
+{"current_steps": 1000, "total_steps": 1515, "loss": 0.2999, "learning_rate": 5.476299244830734e-06, "epoch": 1.9770171753367107, "percentage": 66.01, "elapsed_time": "3:22:15", "remaining_time": "1:44:09"}
+{"current_steps": 1000, "total_steps": 1515, "eval_loss": 0.3708229959011078, "epoch": 1.9770171753367107, "percentage": 66.01, "elapsed_time": "3:23:37", "remaining_time": "1:44:51"}
+{"current_steps": 1010, "total_steps": 1515, "loss": 0.3066, "learning_rate": 5.28662203925562e-06, "epoch": 1.9967873470900779, "percentage": 66.67, "elapsed_time": "3:26:12", "remaining_time": "1:43:06"}
+{"current_steps": 1020, "total_steps": 1515, "loss": 0.2574, "learning_rate": 5.099100452011876e-06, "epoch": 2.016557518843445, "percentage": 67.33, "elapsed_time": "3:28:10", "remaining_time": "1:41:01"}
+{"current_steps": 1030, "total_steps": 1515, "loss": 0.2297, "learning_rate": 4.9138202443039625e-06, "epoch": 2.036327690596812, "percentage": 67.99, "elapsed_time": "3:30:18", "remaining_time": "1:39:01"}
+{"current_steps": 1040, "total_steps": 1515, "loss": 0.2445, "learning_rate": 4.730866152262841e-06, "epoch": 2.056097862350179, "percentage": 68.65, "elapsed_time": "3:32:14", "remaining_time": "1:36:56"}
+{"current_steps": 1050, "total_steps": 1515, "loss": 0.2505, "learning_rate": 4.550321848192731e-06, "epoch": 2.0758680341035465, "percentage": 69.31, "elapsed_time": "3:34:12", "remaining_time": "1:34:51"}
+{"current_steps": 1060, "total_steps": 1515, "loss": 0.2494, "learning_rate": 4.3722699023043634e-06, "epoch": 2.0956382058569134, "percentage": 69.97, "elapsed_time": "3:36:10", "remaining_time": "1:32:47"}
+{"current_steps": 1070, "total_steps": 1515, "loss": 0.239, "learning_rate": 4.196791744952291e-06, "epoch": 2.1154083776102803, "percentage": 70.63, "elapsed_time": "3:38:09", "remaining_time": "1:30:43"}
+{"current_steps": 1080, "total_steps": 1515, "loss": 0.2463, "learning_rate": 4.023967629393498e-06, "epoch": 2.1351785493636477, "percentage": 71.29, "elapsed_time": "3:40:19", "remaining_time": "1:28:44"}
+{"current_steps": 1090, "total_steps": 1515, "loss": 0.2455, "learning_rate": 3.853876595084332e-06, "epoch": 2.1549487211170146, "percentage": 71.95, "elapsed_time": "3:42:18", "remaining_time": "1:26:40"}
+{"current_steps": 1100, "total_steps": 1515, "loss": 0.2406, "learning_rate": 3.68659643153258e-06, "epoch": 2.174718892870382, "percentage": 72.61, "elapsed_time": "3:44:21", "remaining_time": "1:24:38"}
+{"current_steps": 1110, "total_steps": 1515, "loss": 0.2333, "learning_rate": 3.5222036427211704e-06, "epoch": 2.194489064623749, "percentage": 73.27, "elapsed_time": "3:46:30", "remaining_time": "1:22:38"}
+{"current_steps": 1120, "total_steps": 1515, "loss": 0.2397, "learning_rate": 3.360773412119801e-06, "epoch": 2.214259236377116, "percentage": 73.93, "elapsed_time": "3:48:26", "remaining_time": "1:20:34"}
+{"current_steps": 1130, "total_steps": 1515, "loss": 0.2394, "learning_rate": 3.202379568300511e-06, "epoch": 2.2340294081304832, "percentage": 74.59, "elapsed_time": "3:50:22", "remaining_time": "1:18:29"}
+{"current_steps": 1140, "total_steps": 1515, "loss": 0.2319, "learning_rate": 3.0470945511728856e-06, "epoch": 2.25379957988385, "percentage": 75.25, "elapsed_time": "3:52:14", "remaining_time": "1:16:23"}
+{"current_steps": 1150, "total_steps": 1515, "loss": 0.2461, "learning_rate": 2.8949893788543505e-06, "epoch": 2.2735697516372175, "percentage": 75.91, "elapsed_time": "3:54:16", "remaining_time": "1:14:21"}
+{"current_steps": 1160, "total_steps": 1515, "loss": 0.2391, "learning_rate": 2.74613361519073e-06, "epoch": 2.2933399233905845, "percentage": 76.57, "elapsed_time": "3:56:14", "remaining_time": "1:12:17"}
+{"current_steps": 1170, "total_steps": 1515, "loss": 0.2532, "learning_rate": 2.60059533794187e-06, "epoch": 2.3131100951439514, "percentage": 77.23, "elapsed_time": "3:58:16", "remaining_time": "1:10:15"}
+{"current_steps": 1180, "total_steps": 1515, "loss": 0.2442, "learning_rate": 2.458441107646935e-06, "epoch": 2.3328802668973188, "percentage": 77.89, "elapsed_time": "4:00:18", "remaining_time": "1:08:13"}
+{"current_steps": 1190, "total_steps": 1515, "loss": 0.2374, "learning_rate": 2.3197359371835802e-06, "epoch": 2.3526504386506857, "percentage": 78.55, "elapsed_time": "4:02:20", "remaining_time": "1:06:11"}
+{"current_steps": 1200, "total_steps": 1515, "loss": 0.2383, "learning_rate": 2.1845432620349338e-06, "epoch": 2.372420610404053, "percentage": 79.21, "elapsed_time": "4:04:26", "remaining_time": "1:04:09"}
+{"current_steps": 1210, "total_steps": 1515, "loss": 0.2388, "learning_rate": 2.0529249112779824e-06, "epoch": 2.39219078215742, "percentage": 79.87, "elapsed_time": "4:06:28", "remaining_time": "1:02:07"}
+{"current_steps": 1220, "total_steps": 1515, "loss": 0.2437, "learning_rate": 1.924941079306647e-06, "epoch": 2.411960953910787, "percentage": 80.53, "elapsed_time": "4:08:32", "remaining_time": "1:00:05"}
+{"current_steps": 1230, "total_steps": 1515, "loss": 0.2482, "learning_rate": 1.8006502983024332e-06, "epoch": 2.4317311256641543, "percentage": 81.19, "elapsed_time": "4:10:31", "remaining_time": "0:58:02"}
+{"current_steps": 1240, "total_steps": 1515, "loss": 0.2313, "learning_rate": 1.6801094114653082e-06, "epoch": 2.451501297417521, "percentage": 81.85, "elapsed_time": "4:12:30", "remaining_time": "0:56:00"}
+{"current_steps": 1250, "total_steps": 1515, "loss": 0.2403, "learning_rate": 1.5633735470170108e-06, "epoch": 2.4712714691708886, "percentage": 82.51, "elapsed_time": "4:14:44", "remaining_time": "0:54:00"}
+{"current_steps": 1260, "total_steps": 1515, "loss": 0.2442, "learning_rate": 1.4504960929886746e-06, "epoch": 2.4910416409242555, "percentage": 83.17, "elapsed_time": "4:16:46", "remaining_time": "0:51:57"}
+{"current_steps": 1270, "total_steps": 1515, "loss": 0.239, "learning_rate": 1.3415286728043374e-06, "epoch": 2.5108118126776224, "percentage": 83.83, "elapsed_time": "4:18:39", "remaining_time": "0:49:54"}
+{"current_steps": 1280, "total_steps": 1515, "loss": 0.2385, "learning_rate": 1.2365211216714523e-06, "epoch": 2.53058198443099, "percentage": 84.49, "elapsed_time": "4:20:41", "remaining_time": "0:47:51"}
+{"current_steps": 1290, "total_steps": 1515, "loss": 0.2482, "learning_rate": 1.135521463789232e-06, "epoch": 2.5503521561843567, "percentage": 85.15, "elapsed_time": "4:22:35", "remaining_time": "0:45:48"}
+{"current_steps": 1300, "total_steps": 1515, "loss": 0.2381, "learning_rate": 1.0385758903852406e-06, "epoch": 2.570122327937724, "percentage": 85.81, "elapsed_time": "4:24:36", "remaining_time": "0:43:45"}
+{"current_steps": 1310, "total_steps": 1515, "loss": 0.2405, "learning_rate": 9.457287385902836e-07, "epoch": 2.589892499691091, "percentage": 86.47, "elapsed_time": "4:26:34", "remaining_time": "0:41:43"}
+{"current_steps": 1320, "total_steps": 1515, "loss": 0.2304, "learning_rate": 8.570224711612385e-07, "epoch": 2.6096626714444584, "percentage": 87.13, "elapsed_time": "4:28:30", "remaining_time": "0:39:39"}
+{"current_steps": 1330, "total_steps": 1515, "loss": 0.2259, "learning_rate": 7.724976570611254e-07, "epoch": 2.6294328431978253, "percentage": 87.79, "elapsed_time": "4:30:25", "remaining_time": "0:37:36"}
+{"current_steps": 1340, "total_steps": 1515, "loss": 0.2318, "learning_rate": 6.921929529052729e-07, "epoch": 2.6492030149511923, "percentage": 88.45, "elapsed_time": "4:32:27", "remaining_time": "0:35:34"}
+{"current_steps": 1350, "total_steps": 1515, "loss": 0.2377, "learning_rate": 6.161450852820872e-07, "epoch": 2.6689731867045596, "percentage": 89.11, "elapsed_time": "4:34:24", "remaining_time": "0:33:32"}
+{"current_steps": 1360, "total_steps": 1515, "loss": 0.2402, "learning_rate": 5.443888339565018e-07, "epoch": 2.6887433584579266, "percentage": 89.77, "elapsed_time": "4:36:35", "remaining_time": "0:31:31"}
+{"current_steps": 1370, "total_steps": 1515, "loss": 0.2401, "learning_rate": 4.769570159637782e-07, "epoch": 2.7085135302112935, "percentage": 90.43, "elapsed_time": "4:38:28", "remaining_time": "0:29:28"}
+{"current_steps": 1380, "total_steps": 1515, "loss": 0.2306, "learning_rate": 4.138804706009547e-07, "epoch": 2.728283701964661, "percentage": 91.09, "elapsed_time": "4:40:32", "remaining_time": "0:27:26"}
+{"current_steps": 1390, "total_steps": 1515, "loss": 0.2407, "learning_rate": 3.5518804532278736e-07, "epoch": 2.748053873718028, "percentage": 91.75, "elapsed_time": "4:42:22", "remaining_time": "0:25:23"}
+{"current_steps": 1400, "total_steps": 1515, "loss": 0.227, "learning_rate": 3.0090658254864013e-07, "epoch": 2.767824045471395, "percentage": 92.41, "elapsed_time": "4:44:24", "remaining_time": "0:23:21"}
+{"current_steps": 1410, "total_steps": 1515, "loss": 0.2306, "learning_rate": 2.5106090738636014e-07, "epoch": 2.787594217224762, "percentage": 93.07, "elapsed_time": "4:46:27", "remaining_time": "0:21:19"}
+{"current_steps": 1420, "total_steps": 1515, "loss": 0.242, "learning_rate": 2.0567381627875639e-07, "epoch": 2.8073643889781295, "percentage": 93.73, "elapsed_time": "4:48:30", "remaining_time": "0:19:18"}
+{"current_steps": 1430, "total_steps": 1515, "loss": 0.2309, "learning_rate": 1.6476606657786386e-07, "epoch": 2.8271345607314964, "percentage": 94.39, "elapsed_time": "4:50:28", "remaining_time": "0:17:15"}
+{"current_steps": 1440, "total_steps": 1515, "loss": 0.2287, "learning_rate": 1.2835636705176403e-07, "epoch": 2.8469047324848633, "percentage": 95.05, "elapsed_time": "4:52:21", "remaining_time": "0:15:13"}
+{"current_steps": 1450, "total_steps": 1515, "loss": 0.2469, "learning_rate": 9.64613693283123e-08, "epoch": 2.8666749042382307, "percentage": 95.71, "elapsed_time": "4:54:24", "remaining_time": "0:13:11"}
+{"current_steps": 1460, "total_steps": 1515, "loss": 0.2458, "learning_rate": 6.909566027966996e-08, "epoch": 2.8864450759915976, "percentage": 96.37, "elapsed_time": "4:56:19", "remaining_time": "0:11:09"}
+{"current_steps": 1470, "total_steps": 1515, "loss": 0.2456, "learning_rate": 4.6271755351144923e-08, "epoch": 2.9062152477449645, "percentage": 97.03, "elapsed_time": "4:58:22", "remaining_time": "0:09:08"}
+{"current_steps": 1480, "total_steps": 1515, "loss": 0.2413, "learning_rate": 2.8000092837368042e-08, "epoch": 2.925985419498332, "percentage": 97.69, "elapsed_time": "5:00:13", "remaining_time": "0:07:06"}
+{"current_steps": 1490, "total_steps": 1515, "loss": 0.2388, "learning_rate": 1.4289029108436325e-08, "epoch": 2.945755591251699, "percentage": 98.35, "elapsed_time": "5:02:13", "remaining_time": "0:05:04"}
+{"current_steps": 1500, "total_steps": 1515, "loss": 0.2454, "learning_rate": 5.14483478821437e-09, "epoch": 2.965525763005066, "percentage": 99.01, "elapsed_time": "5:04:19", "remaining_time": "0:03:02"}
+{"current_steps": 1500, "total_steps": 1515, "eval_loss": 0.37777501344680786, "epoch": 2.965525763005066, "percentage": 99.01, "elapsed_time": "5:05:41", "remaining_time": "0:03:03"}
+{"current_steps": 1510, "total_steps": 1515, "loss": 0.2349, "learning_rate": 5.716918865195098e-10, "epoch": 2.985295934758433, "percentage": 99.67, "elapsed_time": "5:08:31", "remaining_time": "0:01:01"}
+{"current_steps": 1515, "total_steps": 1515, "epoch": 2.995181020635117, "percentage": 100.0, "elapsed_time": "5:10:14", "remaining_time": "0:00:00"}

trainer_state.json ADDED Viewed

	@@ -0,0 +1,1123 @@

+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.995181020635117,
+  "eval_steps": 500,
+  "global_step": 1515,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.019770171753367108,
+      "grad_norm": 8.273716970639242,
+      "learning_rate": 4.347826086956522e-06,
+      "loss": 1.4192,
+      "step": 10
+    },
+    {
+      "epoch": 0.039540343506734216,
+      "grad_norm": 2.348437423787933,
+      "learning_rate": 8.695652173913044e-06,
+      "loss": 0.9179,
+      "step": 20
+    },
+    {
+      "epoch": 0.059310515260101324,
+      "grad_norm": 2.0183151255256266,
+      "learning_rate": 1.3043478260869566e-05,
+      "loss": 0.6842,
+      "step": 30
+    },
+    {
+      "epoch": 0.07908068701346843,
+      "grad_norm": 1.7096945603933598,
+      "learning_rate": 1.739130434782609e-05,
+      "loss": 0.5843,
+      "step": 40
+    },
+    {
+      "epoch": 0.09885085876683554,
+      "grad_norm": 1.2099539800951016,
+      "learning_rate": 1.999963411593758e-05,
+      "loss": 0.5283,
+      "step": 50
+    },
+    {
+      "epoch": 0.11862103052020265,
+      "grad_norm": 1.539237571955243,
+      "learning_rate": 1.9995518227714435e-05,
+      "loss": 0.513,
+      "step": 60
+    },
+    {
+      "epoch": 0.13839120227356974,
+      "grad_norm": 0.9407018526102695,
+      "learning_rate": 1.9986830984827473e-05,
+      "loss": 0.498,
+      "step": 70
+    },
+    {
+      "epoch": 0.15816137402693686,
+      "grad_norm": 0.8754091153209901,
+      "learning_rate": 1.9973576360304135e-05,
+      "loss": 0.4801,
+      "step": 80
+    },
+    {
+      "epoch": 0.17793154578030396,
+      "grad_norm": 0.8624602134273651,
+      "learning_rate": 1.9955760416020185e-05,
+      "loss": 0.4764,
+      "step": 90
+    },
+    {
+      "epoch": 0.19770171753367108,
+      "grad_norm": 0.8628999100665943,
+      "learning_rate": 1.9933391299927346e-05,
+      "loss": 0.4707,
+      "step": 100
+    },
+    {
+      "epoch": 0.21747188928703817,
+      "grad_norm": 0.8080598017374558,
+      "learning_rate": 1.9906479242326927e-05,
+      "loss": 0.4438,
+      "step": 110
+    },
+    {
+      "epoch": 0.2372420610404053,
+      "grad_norm": 0.7079645225164415,
+      "learning_rate": 1.987503655119108e-05,
+      "loss": 0.4507,
+      "step": 120
+    },
+    {
+      "epoch": 0.2570122327937724,
+      "grad_norm": 0.8311840484882397,
+      "learning_rate": 1.9839077606533878e-05,
+      "loss": 0.4489,
+      "step": 130
+    },
+    {
+      "epoch": 0.2767824045471395,
+      "grad_norm": 0.8147624642320691,
+      "learning_rate": 1.9798618853834747e-05,
+      "loss": 0.4507,
+      "step": 140
+    },
+    {
+      "epoch": 0.29655257630050663,
+      "grad_norm": 0.8371409045137015,
+      "learning_rate": 1.975367879651728e-05,
+      "loss": 0.4542,
+      "step": 150
+    },
+    {
+      "epoch": 0.3163227480538737,
+      "grad_norm": 0.7237155579293956,
+      "learning_rate": 1.970427798748689e-05,
+      "loss": 0.439,
+      "step": 160
+    },
+    {
+      "epoch": 0.3360929198072408,
+      "grad_norm": 0.6804376236541478,
+      "learning_rate": 1.965043901973111e-05,
+      "loss": 0.4281,
+      "step": 170
+    },
+    {
+      "epoch": 0.3558630915606079,
+      "grad_norm": 0.7087718772894039,
+      "learning_rate": 1.9592186515986932e-05,
+      "loss": 0.4255,
+      "step": 180
+    },
+    {
+      "epoch": 0.37563326331397506,
+      "grad_norm": 0.6404671722237,
+      "learning_rate": 1.9529547117479802e-05,
+      "loss": 0.4275,
+      "step": 190
+    },
+    {
+      "epoch": 0.39540343506734216,
+      "grad_norm": 0.6613559685528836,
+      "learning_rate": 1.9462549471739546e-05,
+      "loss": 0.4391,
+      "step": 200
+    },
+    {
+      "epoch": 0.41517360682070925,
+      "grad_norm": 0.7702640494158857,
+      "learning_rate": 1.939122421949868e-05,
+      "loss": 0.4316,
+      "step": 210
+    },
+    {
+      "epoch": 0.43494377857407635,
+      "grad_norm": 0.6687391965346136,
+      "learning_rate": 1.931560398067915e-05,
+      "loss": 0.4309,
+      "step": 220
+    },
+    {
+      "epoch": 0.4547139503274435,
+      "grad_norm": 0.6455592214289195,
+      "learning_rate": 1.923572333947395e-05,
+      "loss": 0.4228,
+      "step": 230
+    },
+    {
+      "epoch": 0.4744841220808106,
+      "grad_norm": 0.6638256696831698,
+      "learning_rate": 1.9151618828530347e-05,
+      "loss": 0.4132,
+      "step": 240
+    },
+    {
+      "epoch": 0.4942542938341777,
+      "grad_norm": 0.6035529106490282,
+      "learning_rate": 1.9063328912242047e-05,
+      "loss": 0.4201,
+      "step": 250
+    },
+    {
+      "epoch": 0.5140244655875448,
+      "grad_norm": 0.6181775363815968,
+      "learning_rate": 1.8970893969157852e-05,
+      "loss": 0.4175,
+      "step": 260
+    },
+    {
+      "epoch": 0.5337946373409119,
+      "grad_norm": 0.6133100135259807,
+      "learning_rate": 1.8874356273514934e-05,
+      "loss": 0.4063,
+      "step": 270
+    },
+    {
+      "epoch": 0.553564809094279,
+      "grad_norm": 0.5966662360815947,
+      "learning_rate": 1.8773759975905098e-05,
+      "loss": 0.4218,
+      "step": 280
+    },
+    {
+      "epoch": 0.5733349808476461,
+      "grad_norm": 0.6703060027446681,
+      "learning_rate": 1.8669151083082955e-05,
+      "loss": 0.4134,
+      "step": 290
+    },
+    {
+      "epoch": 0.5931051526010133,
+      "grad_norm": 0.6252415388725653,
+      "learning_rate": 1.8560577436925144e-05,
+      "loss": 0.3988,
+      "step": 300
+    },
+    {
+      "epoch": 0.6128753243543803,
+      "grad_norm": 0.6060817440235046,
+      "learning_rate": 1.8448088692550337e-05,
+      "loss": 0.4175,
+      "step": 310
+    },
+    {
+      "epoch": 0.6326454961077475,
+      "grad_norm": 0.6593209743601871,
+      "learning_rate": 1.833173629560993e-05,
+      "loss": 0.409,
+      "step": 320
+    },
+    {
+      "epoch": 0.6524156678611145,
+      "grad_norm": 0.5982036518358108,
+      "learning_rate": 1.8211573458759854e-05,
+      "loss": 0.3997,
+      "step": 330
+    },
+    {
+      "epoch": 0.6721858396144816,
+      "grad_norm": 0.5833507470903769,
+      "learning_rate": 1.80876551373243e-05,
+      "loss": 0.4015,
+      "step": 340
+    },
+    {
+      "epoch": 0.6919560113678488,
+      "grad_norm": 0.6279313755216596,
+      "learning_rate": 1.7960038004162426e-05,
+      "loss": 0.4048,
+      "step": 350
+    },
+    {
+      "epoch": 0.7117261831212158,
+      "grad_norm": 0.6685300488229989,
+      "learning_rate": 1.7828780423749536e-05,
+      "loss": 0.4075,
+      "step": 360
+    },
+    {
+      "epoch": 0.731496354874583,
+      "grad_norm": 0.574309905920571,
+      "learning_rate": 1.769394242548469e-05,
+      "loss": 0.4102,
+      "step": 370
+    },
+    {
+      "epoch": 0.7512665266279501,
+      "grad_norm": 0.5657263244028015,
+      "learning_rate": 1.755558567623678e-05,
+      "loss": 0.396,
+      "step": 380
+    },
+    {
+      "epoch": 0.7710366983813172,
+      "grad_norm": 0.5797623404841392,
+      "learning_rate": 1.741377345214184e-05,
+      "loss": 0.4063,
+      "step": 390
+    },
+    {
+      "epoch": 0.7908068701346843,
+      "grad_norm": 0.5734761888945618,
+      "learning_rate": 1.7268570609664253e-05,
+      "loss": 0.4066,
+      "step": 400
+    },
+    {
+      "epoch": 0.8105770418880514,
+      "grad_norm": 0.5725087962478311,
+      "learning_rate": 1.71200435559353e-05,
+      "loss": 0.3848,
+      "step": 410
+    },
+    {
+      "epoch": 0.8303472136414185,
+      "grad_norm": 0.6574852860839745,
+      "learning_rate": 1.6968260218382484e-05,
+      "loss": 0.3882,
+      "step": 420
+    },
+    {
+      "epoch": 0.8501173853947857,
+      "grad_norm": 0.6526897086539006,
+      "learning_rate": 1.6813290013663573e-05,
+      "loss": 0.3957,
+      "step": 430
+    },
+    {
+      "epoch": 0.8698875571481527,
+      "grad_norm": 0.6101372561347892,
+      "learning_rate": 1.665520381591955e-05,
+      "loss": 0.3958,
+      "step": 440
+    },
+    {
+      "epoch": 0.8896577289015198,
+      "grad_norm": 0.6102719326578399,
+      "learning_rate": 1.6494073924361012e-05,
+      "loss": 0.3846,
+      "step": 450
+    },
+    {
+      "epoch": 0.909427900654887,
+      "grad_norm": 0.6284095546066177,
+      "learning_rate": 1.6329974030202803e-05,
+      "loss": 0.3925,
+      "step": 460
+    },
+    {
+      "epoch": 0.929198072408254,
+      "grad_norm": 0.5573501907726315,
+      "learning_rate": 1.6162979182962068e-05,
+      "loss": 0.3806,
+      "step": 470
+    },
+    {
+      "epoch": 0.9489682441616212,
+      "grad_norm": 0.6587508652316405,
+      "learning_rate": 1.5993165756135035e-05,
+      "loss": 0.4075,
+      "step": 480
+    },
+    {
+      "epoch": 0.9687384159149882,
+      "grad_norm": 0.5309276250787578,
+      "learning_rate": 1.5820611412268366e-05,
+      "loss": 0.3771,
+      "step": 490
+    },
+    {
+      "epoch": 0.9885085876683554,
+      "grad_norm": 0.5555801576940601,
+      "learning_rate": 1.564539506744091e-05,
+      "loss": 0.3869,
+      "step": 500
+    },
+    {
+      "epoch": 0.9885085876683554,
+      "eval_loss": 0.39637455344200134,
+      "eval_runtime": 81.285,
+      "eval_samples_per_second": 83.841,
+      "eval_steps_per_second": 5.241,
+      "step": 500
+    },
+    {
+      "epoch": 1.0082787594217224,
+      "grad_norm": 0.5067307545314473,
+      "learning_rate": 1.546759685517219e-05,
+      "loss": 0.356,
+      "step": 510
+    },
+    {
+      "epoch": 1.0280489311750896,
+      "grad_norm": 0.5618999195126448,
+      "learning_rate": 1.528729808977412e-05,
+      "loss": 0.3228,
+      "step": 520
+    },
+    {
+      "epoch": 1.0478191029284567,
+      "grad_norm": 0.5098273070317775,
+      "learning_rate": 1.5104581229162674e-05,
+      "loss": 0.3182,
+      "step": 530
+    },
+    {
+      "epoch": 1.0675892746818239,
+      "grad_norm": 0.5533254988756643,
+      "learning_rate": 1.4919529837146529e-05,
+      "loss": 0.3171,
+      "step": 540
+    },
+    {
+      "epoch": 1.087359446435191,
+      "grad_norm": 0.5964907411663062,
+      "learning_rate": 1.4732228545209993e-05,
+      "loss": 0.3202,
+      "step": 550
+    },
+    {
+      "epoch": 1.107129618188558,
+      "grad_norm": 0.5065864215097721,
+      "learning_rate": 1.4542763013807568e-05,
+      "loss": 0.3121,
+      "step": 560
+    },
+    {
+      "epoch": 1.126899789941925,
+      "grad_norm": 0.5343714773450696,
+      "learning_rate": 1.4351219893187984e-05,
+      "loss": 0.3179,
+      "step": 570
+    },
+    {
+      "epoch": 1.1466699616952922,
+      "grad_norm": 0.5681605326118392,
+      "learning_rate": 1.4157686783765534e-05,
+      "loss": 0.3112,
+      "step": 580
+    },
+    {
+      "epoch": 1.1664401334486594,
+      "grad_norm": 0.5604875931807674,
+      "learning_rate": 1.3962252196056847e-05,
+      "loss": 0.32,
+      "step": 590
+    },
+    {
+      "epoch": 1.1862103052020265,
+      "grad_norm": 0.5551583789751445,
+      "learning_rate": 1.3765005510201466e-05,
+      "loss": 0.3156,
+      "step": 600
+    },
+    {
+      "epoch": 1.2059804769553935,
+      "grad_norm": 0.5770289171897912,
+      "learning_rate": 1.3566036935084692e-05,
+      "loss": 0.3196,
+      "step": 610
+    },
+    {
+      "epoch": 1.2257506487087606,
+      "grad_norm": 0.5283055302681036,
+      "learning_rate": 1.3365437467081406e-05,
+      "loss": 0.3218,
+      "step": 620
+    },
+    {
+      "epoch": 1.2455208204621278,
+      "grad_norm": 0.5056609484631435,
+      "learning_rate": 1.3163298848439758e-05,
+      "loss": 0.315,
+      "step": 630
+    },
+    {
+      "epoch": 1.265290992215495,
+      "grad_norm": 0.555127018751701,
+      "learning_rate": 1.2959713525323725e-05,
+      "loss": 0.3221,
+      "step": 640
+    },
+    {
+      "epoch": 1.285061163968862,
+      "grad_norm": 0.5130249143640312,
+      "learning_rate": 1.2754774605533728e-05,
+      "loss": 0.3111,
+      "step": 650
+    },
+    {
+      "epoch": 1.3048313357222292,
+      "grad_norm": 0.519020033436049,
+      "learning_rate": 1.2548575815924689e-05,
+      "loss": 0.3151,
+      "step": 660
+    },
+    {
+      "epoch": 1.3246015074755961,
+      "grad_norm": 0.5149083765226259,
+      "learning_rate": 1.234121145954094e-05,
+      "loss": 0.31,
+      "step": 670
+    },
+    {
+      "epoch": 1.3443716792289633,
+      "grad_norm": 0.5301240336350007,
+      "learning_rate": 1.2132776372487634e-05,
+      "loss": 0.3139,
+      "step": 680
+    },
+    {
+      "epoch": 1.3641418509823304,
+      "grad_norm": 0.4880188656600289,
+      "learning_rate": 1.1923365880558345e-05,
+      "loss": 0.3147,
+      "step": 690
+    },
+    {
+      "epoch": 1.3839120227356976,
+      "grad_norm": 0.5098313222505657,
+      "learning_rate": 1.1713075755638748e-05,
+      "loss": 0.3153,
+      "step": 700
+    },
+    {
+      "epoch": 1.4036821944890647,
+      "grad_norm": 0.5563900244184181,
+      "learning_rate": 1.1502002171906248e-05,
+      "loss": 0.3137,
+      "step": 710
+    },
+    {
+      "epoch": 1.4234523662424317,
+      "grad_norm": 0.5308336365319577,
+      "learning_rate": 1.1290241661845649e-05,
+      "loss": 0.301,
+      "step": 720
+    },
+    {
+      "epoch": 1.4432225379957988,
+      "grad_norm": 0.5347937895279813,
+      "learning_rate": 1.1077891072100944e-05,
+      "loss": 0.3114,
+      "step": 730
+    },
+    {
+      "epoch": 1.462992709749166,
+      "grad_norm": 0.5353226891153153,
+      "learning_rate": 1.0865047519183411e-05,
+      "loss": 0.3132,
+      "step": 740
+    },
+    {
+      "epoch": 1.482762881502533,
+      "grad_norm": 0.5827716787607696,
+      "learning_rate": 1.065180834505634e-05,
+      "loss": 0.3032,
+      "step": 750
+    },
+    {
+      "epoch": 1.5025330532559003,
+      "grad_norm": 0.5898123869988975,
+      "learning_rate": 1.0438271072616584e-05,
+      "loss": 0.3113,
+      "step": 760
+    },
+    {
+      "epoch": 1.5223032250092672,
+      "grad_norm": 0.5298973225294878,
+      "learning_rate": 1.022453336109341e-05,
+      "loss": 0.3219,
+      "step": 770
+    },
+    {
+      "epoch": 1.5420733967626343,
+      "grad_norm": 0.5690896784690122,
+      "learning_rate": 1.001069296138499e-05,
+      "loss": 0.3134,
+      "step": 780
+    },
+    {
+      "epoch": 1.5618435685160015,
+      "grad_norm": 0.5827223568466493,
+      "learning_rate": 9.796847671352948e-06,
+      "loss": 0.3126,
+      "step": 790
+    },
+    {
+      "epoch": 1.5816137402693686,
+      "grad_norm": 0.5429937702192498,
+      "learning_rate": 9.583095291095454e-06,
+      "loss": 0.3039,
+      "step": 800
+    },
+    {
+      "epoch": 1.6013839120227358,
+      "grad_norm": 0.5498582424119081,
+      "learning_rate": 9.36953357821928e-06,
+      "loss": 0.3043,
+      "step": 810
+    },
+    {
+      "epoch": 1.6211540837761027,
+      "grad_norm": 0.4994859445298306,
+      "learning_rate": 9.156260203131299e-06,
+      "loss": 0.296,
+      "step": 820
+    },
+    {
+      "epoch": 1.64092425552947,
+      "grad_norm": 0.5154683199933622,
+      "learning_rate": 8.943372704369844e-06,
+      "loss": 0.2962,
+      "step": 830
+    },
+    {
+      "epoch": 1.660694427282837,
+      "grad_norm": 0.5167087299820147,
+      "learning_rate": 8.730968443996411e-06,
+      "loss": 0.3129,
+      "step": 840
+    },
+    {
+      "epoch": 1.6804645990362042,
+      "grad_norm": 0.5490107000824208,
+      "learning_rate": 8.519144563068037e-06,
+      "loss": 0.3119,
+      "step": 850
+    },
+    {
+      "epoch": 1.7002347707895713,
+      "grad_norm": 0.540920173955098,
+      "learning_rate": 8.30799793721077e-06,
+      "loss": 0.3056,
+      "step": 860
+    },
+    {
+      "epoch": 1.7200049425429382,
+      "grad_norm": 0.5237649749912278,
+      "learning_rate": 8.097625132314529e-06,
+      "loss": 0.311,
+      "step": 870
+    },
+    {
+      "epoch": 1.7397751142963056,
+      "grad_norm": 0.6272718536197235,
+      "learning_rate": 7.888122360369617e-06,
+      "loss": 0.3047,
+      "step": 880
+    },
+    {
+      "epoch": 1.7595452860496725,
+      "grad_norm": 0.5296663362064814,
+      "learning_rate": 7.67958543546509e-06,
+      "loss": 0.3011,
+      "step": 890
+    },
+    {
+      "epoch": 1.7793154578030397,
+      "grad_norm": 0.5233109909144185,
+      "learning_rate": 7.472109729969097e-06,
+      "loss": 0.2958,
+      "step": 900
+    },
+    {
+      "epoch": 1.7990856295564068,
+      "grad_norm": 0.5025706432729875,
+      "learning_rate": 7.2657901309112325e-06,
+      "loss": 0.3035,
+      "step": 910
+    },
+    {
+      "epoch": 1.8188558013097738,
+      "grad_norm": 0.5402559569074855,
+      "learning_rate": 7.060720996586878e-06,
+      "loss": 0.3134,
+      "step": 920
+    },
+    {
+      "epoch": 1.8386259730631411,
+      "grad_norm": 0.547259180262811,
+      "learning_rate": 6.85699611340333e-06,
+      "loss": 0.3055,
+      "step": 930
+    },
+    {
+      "epoch": 1.858396144816508,
+      "grad_norm": 0.5197703756039205,
+      "learning_rate": 6.654708652987475e-06,
+      "loss": 0.3043,
+      "step": 940
+    },
+    {
+      "epoch": 1.8781663165698752,
+      "grad_norm": 0.5323204838924858,
+      "learning_rate": 6.453951129574644e-06,
+      "loss": 0.3048,
+      "step": 950
+    },
+    {
+      "epoch": 1.8979364883232424,
+      "grad_norm": 0.47841296810426304,
+      "learning_rate": 6.254815357698118e-06,
+      "loss": 0.3006,
+      "step": 960
+    },
+    {
+      "epoch": 1.9177066600766093,
+      "grad_norm": 0.5083748369122659,
+      "learning_rate": 6.05739241019862e-06,
+      "loss": 0.3025,
+      "step": 970
+    },
+    {
+      "epoch": 1.9374768318299767,
+      "grad_norm": 0.4928502314870966,
+      "learning_rate": 5.861772576573028e-06,
+      "loss": 0.2956,
+      "step": 980
+    },
+    {
+      "epoch": 1.9572470035833436,
+      "grad_norm": 0.48791550019041724,
+      "learning_rate": 5.668045321681328e-06,
+      "loss": 0.3091,
+      "step": 990
+    },
+    {
+      "epoch": 1.9770171753367107,
+      "grad_norm": 0.5066742151391088,
+      "learning_rate": 5.476299244830734e-06,
+      "loss": 0.2999,
+      "step": 1000
+    },
+    {
+      "epoch": 1.9770171753367107,
+      "eval_loss": 0.3708229959011078,
+      "eval_runtime": 81.8274,
+      "eval_samples_per_second": 83.285,
+      "eval_steps_per_second": 5.206,
+      "step": 1000
+    },
+    {
+      "epoch": 1.9967873470900779,
+      "grad_norm": 0.5294994920674526,
+      "learning_rate": 5.28662203925562e-06,
+      "loss": 0.3066,
+      "step": 1010
+    },
+    {
+      "epoch": 2.016557518843445,
+      "grad_norm": 0.5249670268523262,
+      "learning_rate": 5.099100452011876e-06,
+      "loss": 0.2574,
+      "step": 1020
+    },
+    {
+      "epoch": 2.036327690596812,
+      "grad_norm": 0.5209226672918731,
+      "learning_rate": 4.9138202443039625e-06,
+      "loss": 0.2297,
+      "step": 1030
+    },
+    {
+      "epoch": 2.056097862350179,
+      "grad_norm": 0.4884858081963297,
+      "learning_rate": 4.730866152262841e-06,
+      "loss": 0.2445,
+      "step": 1040
+    },
+    {
+      "epoch": 2.0758680341035465,
+      "grad_norm": 0.4742867737071898,
+      "learning_rate": 4.550321848192731e-06,
+      "loss": 0.2505,
+      "step": 1050
+    },
+    {
+      "epoch": 2.0956382058569134,
+      "grad_norm": 0.522161636411317,
+      "learning_rate": 4.3722699023043634e-06,
+      "loss": 0.2494,
+      "step": 1060
+    },
+    {
+      "epoch": 2.1154083776102803,
+      "grad_norm": 0.4240817050624604,
+      "learning_rate": 4.196791744952291e-06,
+      "loss": 0.239,
+      "step": 1070
+    },
+    {
+      "epoch": 2.1351785493636477,
+      "grad_norm": 0.5016013699913245,
+      "learning_rate": 4.023967629393498e-06,
+      "loss": 0.2463,
+      "step": 1080
+    },
+    {
+      "epoch": 2.1549487211170146,
+      "grad_norm": 0.4907508903718958,
+      "learning_rate": 3.853876595084332e-06,
+      "loss": 0.2455,
+      "step": 1090
+    },
+    {
+      "epoch": 2.174718892870382,
+      "grad_norm": 0.4844801070690468,
+      "learning_rate": 3.68659643153258e-06,
+      "loss": 0.2406,
+      "step": 1100
+    },
+    {
+      "epoch": 2.194489064623749,
+      "grad_norm": 0.4805850138516767,
+      "learning_rate": 3.5222036427211704e-06,
+      "loss": 0.2333,
+      "step": 1110
+    },
+    {
+      "epoch": 2.214259236377116,
+      "grad_norm": 0.4850837241450545,
+      "learning_rate": 3.360773412119801e-06,
+      "loss": 0.2397,
+      "step": 1120
+    },
+    {
+      "epoch": 2.2340294081304832,
+      "grad_norm": 0.46778943159588726,
+      "learning_rate": 3.202379568300511e-06,
+      "loss": 0.2394,
+      "step": 1130
+    },
+    {
+      "epoch": 2.25379957988385,
+      "grad_norm": 0.4262009004075529,
+      "learning_rate": 3.0470945511728856e-06,
+      "loss": 0.2319,
+      "step": 1140
+    },
+    {
+      "epoch": 2.2735697516372175,
+      "grad_norm": 0.4910291271379484,
+      "learning_rate": 2.8949893788543505e-06,
+      "loss": 0.2461,
+      "step": 1150
+    },
+    {
+      "epoch": 2.2933399233905845,
+      "grad_norm": 0.5099593229914312,
+      "learning_rate": 2.74613361519073e-06,
+      "loss": 0.2391,
+      "step": 1160
+    },
+    {
+      "epoch": 2.3131100951439514,
+      "grad_norm": 0.47785223375797403,
+      "learning_rate": 2.60059533794187e-06,
+      "loss": 0.2532,
+      "step": 1170
+    },
+    {
+      "epoch": 2.3328802668973188,
+      "grad_norm": 0.48894964709263233,
+      "learning_rate": 2.458441107646935e-06,
+      "loss": 0.2442,
+      "step": 1180
+    },
+    {
+      "epoch": 2.3526504386506857,
+      "grad_norm": 0.4887403938640547,
+      "learning_rate": 2.3197359371835802e-06,
+      "loss": 0.2374,
+      "step": 1190
+    },
+    {
+      "epoch": 2.372420610404053,
+      "grad_norm": 0.5094057476841393,
+      "learning_rate": 2.1845432620349338e-06,
+      "loss": 0.2383,
+      "step": 1200
+    },
+    {
+      "epoch": 2.39219078215742,
+      "grad_norm": 0.4810409572583604,
+      "learning_rate": 2.0529249112779824e-06,
+      "loss": 0.2388,
+      "step": 1210
+    },
+    {
+      "epoch": 2.411960953910787,
+      "grad_norm": 0.506178462808245,
+      "learning_rate": 1.924941079306647e-06,
+      "loss": 0.2437,
+      "step": 1220
+    },
+    {
+      "epoch": 2.4317311256641543,
+      "grad_norm": 0.5101355373820005,
+      "learning_rate": 1.8006502983024332e-06,
+      "loss": 0.2482,
+      "step": 1230
+    },
+    {
+      "epoch": 2.451501297417521,
+      "grad_norm": 0.47636683086858184,
+      "learning_rate": 1.6801094114653082e-06,
+      "loss": 0.2313,
+      "step": 1240
+    },
+    {
+      "epoch": 2.4712714691708886,
+      "grad_norm": 0.43908801679241943,
+      "learning_rate": 1.5633735470170108e-06,
+      "loss": 0.2403,
+      "step": 1250
+    },
+    {
+      "epoch": 2.4910416409242555,
+      "grad_norm": 0.46494594811457074,
+      "learning_rate": 1.4504960929886746e-06,
+      "loss": 0.2442,
+      "step": 1260
+    },
+    {
+      "epoch": 2.5108118126776224,
+      "grad_norm": 0.49463377523743013,
+      "learning_rate": 1.3415286728043374e-06,
+      "loss": 0.239,
+      "step": 1270
+    },
+    {
+      "epoch": 2.53058198443099,
+      "grad_norm": 0.46074492651979493,
+      "learning_rate": 1.2365211216714523e-06,
+      "loss": 0.2385,
+      "step": 1280
+    },
+    {
+      "epoch": 2.5503521561843567,
+      "grad_norm": 0.5218071815372649,
+      "learning_rate": 1.135521463789232e-06,
+      "loss": 0.2482,
+      "step": 1290
+    },
+    {
+      "epoch": 2.570122327937724,
+      "grad_norm": 0.4982910156071213,
+      "learning_rate": 1.0385758903852406e-06,
+      "loss": 0.2381,
+      "step": 1300
+    },
+    {
+      "epoch": 2.589892499691091,
+      "grad_norm": 0.5007115279363157,
+      "learning_rate": 9.457287385902836e-07,
+      "loss": 0.2405,
+      "step": 1310
+    },
+    {
+      "epoch": 2.6096626714444584,
+      "grad_norm": 0.4830944811979268,
+      "learning_rate": 8.570224711612385e-07,
+      "loss": 0.2304,
+      "step": 1320
+    },
+    {
+      "epoch": 2.6294328431978253,
+      "grad_norm": 0.47077203411861107,
+      "learning_rate": 7.724976570611254e-07,
+      "loss": 0.2259,
+      "step": 1330
+    },
+    {
+      "epoch": 2.6492030149511923,
+      "grad_norm": 0.49030170297745246,
+      "learning_rate": 6.921929529052729e-07,
+      "loss": 0.2318,
+      "step": 1340
+    },
+    {
+      "epoch": 2.6689731867045596,
+      "grad_norm": 0.49483419009773855,
+      "learning_rate": 6.161450852820872e-07,
+      "loss": 0.2377,
+      "step": 1350
+    },
+    {
+      "epoch": 2.6887433584579266,
+      "grad_norm": 0.5047141018868604,
+      "learning_rate": 5.443888339565018e-07,
+      "loss": 0.2402,
+      "step": 1360
+    },
+    {
+      "epoch": 2.7085135302112935,
+      "grad_norm": 0.4627659592534385,
+      "learning_rate": 4.769570159637782e-07,
+      "loss": 0.2401,
+      "step": 1370
+    },
+    {
+      "epoch": 2.728283701964661,
+      "grad_norm": 0.5051673776560596,
+      "learning_rate": 4.138804706009547e-07,
+      "loss": 0.2306,
+      "step": 1380
+    },
+    {
+      "epoch": 2.748053873718028,
+      "grad_norm": 0.4697753055417383,
+      "learning_rate": 3.5518804532278736e-07,
+      "loss": 0.2407,
+      "step": 1390
+    },
+    {
+      "epoch": 2.767824045471395,
+      "grad_norm": 0.48255171108333217,
+      "learning_rate": 3.0090658254864013e-07,
+      "loss": 0.227,
+      "step": 1400
+    },
+    {
+      "epoch": 2.787594217224762,
+      "grad_norm": 0.48725984214652784,
+      "learning_rate": 2.5106090738636014e-07,
+      "loss": 0.2306,
+      "step": 1410
+    },
+    {
+      "epoch": 2.8073643889781295,
+      "grad_norm": 0.5176868460783628,
+      "learning_rate": 2.0567381627875639e-07,
+      "loss": 0.242,
+      "step": 1420
+    },
+    {
+      "epoch": 2.8271345607314964,
+      "grad_norm": 0.486977389827133,
+      "learning_rate": 1.6476606657786386e-07,
+      "loss": 0.2309,
+      "step": 1430
+    },
+    {
+      "epoch": 2.8469047324848633,
+      "grad_norm": 0.48480851362499594,
+      "learning_rate": 1.2835636705176403e-07,
+      "loss": 0.2287,
+      "step": 1440
+    },
+    {
+      "epoch": 2.8666749042382307,
+      "grad_norm": 0.506776005156481,
+      "learning_rate": 9.64613693283123e-08,
+      "loss": 0.2469,
+      "step": 1450
+    },
+    {
+      "epoch": 2.8864450759915976,
+      "grad_norm": 0.5370285292991429,
+      "learning_rate": 6.909566027966996e-08,
+      "loss": 0.2458,
+      "step": 1460
+    },
+    {
+      "epoch": 2.9062152477449645,
+      "grad_norm": 0.46946884718156673,
+      "learning_rate": 4.6271755351144923e-08,
+      "loss": 0.2456,
+      "step": 1470
+    },
+    {
+      "epoch": 2.925985419498332,
+      "grad_norm": 0.47309149600612144,
+      "learning_rate": 2.8000092837368042e-08,
+      "loss": 0.2413,
+      "step": 1480
+    },
+    {
+      "epoch": 2.945755591251699,
+      "grad_norm": 0.48781257822426394,
+      "learning_rate": 1.4289029108436325e-08,
+      "loss": 0.2388,
+      "step": 1490
+    },
+    {
+      "epoch": 2.965525763005066,
+      "grad_norm": 0.4929015642135056,
+      "learning_rate": 5.14483478821437e-09,
+      "loss": 0.2454,
+      "step": 1500
+    },
+    {
+      "epoch": 2.965525763005066,
+      "eval_loss": 0.37777501344680786,
+      "eval_runtime": 82.1049,
+      "eval_samples_per_second": 83.004,
+      "eval_steps_per_second": 5.188,
+      "step": 1500
+    },
+    {
+      "epoch": 2.985295934758433,
+      "grad_norm": 0.49492433015567533,
+      "learning_rate": 5.716918865195098e-10,
+      "loss": 0.2349,
+      "step": 1510
+    },
+    {
+      "epoch": 2.995181020635117,
+      "step": 1515,
+      "total_flos": 504886288187392.0,
+      "train_loss": 0.33701497725527674,
+      "train_runtime": 18615.961,
+      "train_samples_per_second": 20.865,
+      "train_steps_per_second": 0.081
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1515,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 504886288187392.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:3768f88178a75966fa687591fa8fb2a1d271eb4b4e6ae283963a64c0e961ac10
+size 7288

vocab.json ADDED Viewed

The diff for this file is too large to render. See raw diff