Upload folder using huggingface_hub

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 original_model/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+original_model/results_\['boolq',[[:space:]]'lambada_openai',[[:space:]]'logiqa',[[:space:]]'mmlu',[[:space:]]'openbookqa',[[:space:]]'hellaswag'\]_raw.json filter=lfs diff=lfs merge=lfs -text

original_model/config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "_name_or_path": "/mnt/scratch/bborisov/models/perf_models/experts-7/original_model",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -24,6 +24,7 @@
   "num_experts": 7,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
+  "output_router_logits": false,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": {
@@ -34,8 +35,9 @@
     "rope_type": "llama3"
   },
   "rope_theta": 500000.0,
+  "router_loss_coef": 0,
   "tie_word_embeddings": false,
-  "torch_dtype": "float16",
+  "torch_dtype": "bfloat16",
   "transformers_version": "4.46.2",
   "use_cache": true,
   "vocab_size": 128256

original_model/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "transformers_version": "4.46.2"
+}

original_model/model-00001-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3a736a91621a8a672fd3d180650bf386210f6c2cd5444b363a9238f2614cd6d
+size 4976620688

original_model/model-00002-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92cc6ac0de9b025124bcb4faaa0015b907199e8967bd62ea83f5f65807d46d48
+size 4982987152

original_model/model-00003-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cfa9f85bb0e0d5210770ae5f51947ec0f7ab7b5c10da0b0223bdfe39086148b
+size 4982987144

original_model/model-00004-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98e0240d98a8d771959690ad7f1052e856c91264767fc599f52f9c2917142fca
+size 4982987144

original_model/model-00005-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:404d2066a124df3bdfdd23554ee8c20aa45daade69ddd5317f501a3b321c790e
+size 4982987152

original_model/model-00006-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe12e43e339a2a0750f5438c6e8b4da10ff9b8d8ddaca0ec92dce9e5d7ad66d7
+size 4982987184

original_model/model-00007-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5be1c969a112cb3eb2db4d2e758a2eeef218a7cec5078d9fff7ea7358e2aede3
+size 4982987200

original_model/model-00008-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea2d161302078826527b56669e267e048b3260f26d8d5ac8609cd3ad962cb32b
+size 4982987208

original_model/model-00009-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ffd954c81db8266c6c707885ec03365257d34b7e0c1fc31da057e59309889a5
+size 4982987200

original_model/model-00010-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e85afb61a7a2c466308f0818445593980adce5c31702a9cd0ae9adb58bd23946
+size 4982987200

original_model/model-00011-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b1503d26ab96bb7da46503353ac29da829139480681c060751fabdff7a86e73
+size 4982987208

original_model/model-00012-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1d5ae05e5438be6389dae777cc738a5ec0e663366d5770ae1ee4580fc0d9ad8
+size 4982987200

original_model/model-00013-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:743d4bb9c77b5b0a203bd1b97cb4914ceff826c314a5332c00725b1377eed120
+size 4899026584

original_model/model-00014-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:483024037dba23b0cf68d67377670a9d37c9a9165a86605b0696eb5161b1d846
+size 4982987200

original_model/model-00015-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6a631ddb1105438bdaad205e07ea027231d603b93f9c91a3120c84426b8f82d
+size 4982987200

original_model/model-00016-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ec058cf41c4f236bb4a0c92cab81fef06d843d5ced5aa3180679ae63f877f73
+size 4982987200

original_model/model-00017-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3fc9d0c2429655545c7eeb64838464f57886fc7a520de639e35bdf9d0ca9946
+size 4070732088

original_model/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddbf549a445b5f621d4eda765cb0d086c6837ea46c9043cb85e50b4d9e2c00ea
+size 3697406

original_model/results_['boolq', 'lambada_openai', 'logiqa', 'mmlu', 'openbookqa', 'hellaswag'].json

@@ -0,0 +1,345 @@
+{
+  "boolq": {
+    "alias": "boolq",
+    "acc,none": 0.8428134556574923,
+    "acc_stderr,none": 0.006365990015975688
+  },
+  "hellaswag": {
+    "alias": "hellaswag",
+    "acc,none": 0.58424616610237,
+    "acc_stderr,none": 0.004918442328872138,
+    "acc_norm,none": 0.7112129057956582,
+    "acc_norm_stderr,none": 0.004522725412557139
+  },
+  "lambada_openai": {
+    "alias": "lambada_openai",
+    "perplexity,none": 20800.93696726465,
+    "perplexity_stderr,none": 1263.5031629528712,
+    "acc,none": 0.00038812342324859306,
+    "acc_stderr,none": 0.0002744180684505009
+  },
+  "logiqa": {
+    "alias": "logiqa",
+    "acc,none": 0.3210445468509985,
+    "acc_stderr,none": 0.018312456701476167,
+    "acc_norm,none": 0.3225806451612903,
+    "acc_norm_stderr,none": 0.01833543742125178
+  },
+  "mmlu": {
+    "acc,none": 0.6545363908275175,
+    "acc_stderr,none": 0.0038217824226365127,
+    "alias": "mmlu"
+  },
+  "mmlu_humanities": {
+    "acc,none": 0.614240170031881,
+    "acc_stderr,none": 0.006837872482469845,
+    "alias": " - humanities"
+  },
+  "mmlu_formal_logic": {
+    "alias": "  - formal_logic",
+    "acc,none": 0.5317460317460317,
+    "acc_stderr,none": 0.04463112720677173
+  },
+  "mmlu_high_school_european_history": {
+    "alias": "  - high_school_european_history",
+    "acc,none": 0.7272727272727273,
+    "acc_stderr,none": 0.03477691162163663
+  },
+  "mmlu_high_school_us_history": {
+    "alias": "  - high_school_us_history",
+    "acc,none": 0.8137254901960784,
+    "acc_stderr,none": 0.027325470966716357
+  },
+  "mmlu_high_school_world_history": {
+    "alias": "  - high_school_world_history",
+    "acc,none": 0.8607594936708861,
+    "acc_stderr,none": 0.022535526352692684
+  },
+  "mmlu_international_law": {
+    "alias": "  - international_law",
+    "acc,none": 0.7768595041322314,
+    "acc_stderr,none": 0.038007544752287334
+  },
+  "mmlu_jurisprudence": {
+    "alias": "  - jurisprudence",
+    "acc,none": 0.75,
+    "acc_stderr,none": 0.04186091791394607
+  },
+  "mmlu_logical_fallacies": {
+    "alias": "  - logical_fallacies",
+    "acc,none": 0.7730061349693251,
+    "acc_stderr,none": 0.03291099578615763
+  },
+  "mmlu_moral_disputes": {
+    "alias": "  - moral_disputes",
+    "acc,none": 0.6676300578034682,
+    "acc_stderr,none": 0.025361168749688214
+  },
+  "mmlu_moral_scenarios": {
+    "alias": "  - moral_scenarios",
+    "acc,none": 0.4972067039106145,
+    "acc_stderr,none": 0.016722240595491652
+  },
+  "mmlu_philosophy": {
+    "alias": "  - philosophy",
+    "acc,none": 0.7009646302250804,
+    "acc_stderr,none": 0.026003301117885184
+  },
+  "mmlu_prehistory": {
+    "alias": "  - prehistory",
+    "acc,none": 0.7222222222222222,
+    "acc_stderr,none": 0.02492200116888636
+  },
+  "mmlu_professional_law": {
+    "alias": "  - professional_law",
+    "acc,none": 0.4973924380704042,
+    "acc_stderr,none": 0.012770062445433187
+  },
+  "mmlu_world_religions": {
+    "alias": "  - world_religions",
+    "acc,none": 0.8245614035087719,
+    "acc_stderr,none": 0.0291708855007277
+  },
+  "mmlu_other": {
+    "acc,none": 0.7261023495333119,
+    "acc_stderr,none": 0.007739552801425664,
+    "alias": " - other"
+  },
+  "mmlu_business_ethics": {
+    "alias": "  - business_ethics",
+    "acc,none": 0.65,
+    "acc_stderr,none": 0.04793724854411023
+  },
+  "mmlu_clinical_knowledge": {
+    "alias": "  - clinical_knowledge",
+    "acc,none": 0.7433962264150943,
+    "acc_stderr,none": 0.02688064788905203
+  },
+  "mmlu_college_medicine": {
+    "alias": "  - college_medicine",
+    "acc,none": 0.6473988439306358,
+    "acc_stderr,none": 0.03643037168958545
+  },
+  "mmlu_global_facts": {
+    "alias": "  - global_facts",
+    "acc,none": 0.44,
+    "acc_stderr,none": 0.049888765156985884
+  },
+  "mmlu_human_aging": {
+    "alias": "  - human_aging",
+    "acc,none": 0.672645739910314,
+    "acc_stderr,none": 0.0314938467099413
+  },
+  "mmlu_management": {
+    "alias": "  - management",
+    "acc,none": 0.8058252427184466,
+    "acc_stderr,none": 0.03916667762822582
+  },
+  "mmlu_marketing": {
+    "alias": "  - marketing",
+    "acc,none": 0.8418803418803419,
+    "acc_stderr,none": 0.02390232554956044
+  },
+  "mmlu_medical_genetics": {
+    "alias": "  - medical_genetics",
+    "acc,none": 0.75,
+    "acc_stderr,none": 0.04351941398892446
+  },
+  "mmlu_miscellaneous": {
+    "alias": "  - miscellaneous",
+    "acc,none": 0.8173690932311622,
+    "acc_stderr,none": 0.013816335389973074
+  },
+  "mmlu_nutrition": {
+    "alias": "  - nutrition",
+    "acc,none": 0.7679738562091504,
+    "acc_stderr,none": 0.024170840879340925
+  },
+  "mmlu_professional_accounting": {
+    "alias": "  - professional_accounting",
+    "acc,none": 0.5177304964539007,
+    "acc_stderr,none": 0.02980873964223769
+  },
+  "mmlu_professional_medicine": {
+    "alias": "  - professional_medicine",
+    "acc,none": 0.8198529411764706,
+    "acc_stderr,none": 0.0233451636165449
+  },
+  "mmlu_virology": {
+    "alias": "  - virology",
+    "acc,none": 0.536144578313253,
+    "acc_stderr,none": 0.038823108508905954
+  },
+  "mmlu_social_sciences": {
+    "acc,none": 0.7517062073448164,
+    "acc_stderr,none": 0.007628013171255809,
+    "alias": " - social sciences"
+  },
+  "mmlu_econometrics": {
+    "alias": "  - econometrics",
+    "acc,none": 0.5263157894736842,
+    "acc_stderr,none": 0.04697085136647865
+  },
+  "mmlu_high_school_geography": {
+    "alias": "  - high_school_geography",
+    "acc,none": 0.7727272727272727,
+    "acc_stderr,none": 0.029857515673386438
+  },
+  "mmlu_high_school_government_and_politics": {
+    "alias": "  - high_school_government_and_politics",
+    "acc,none": 0.8497409326424871,
+    "acc_stderr,none": 0.025787723180723855
+  },
+  "mmlu_high_school_macroeconomics": {
+    "alias": "  - high_school_macroeconomics",
+    "acc,none": 0.6666666666666666,
+    "acc_stderr,none": 0.023901157979402492
+  },
+  "mmlu_high_school_microeconomics": {
+    "alias": "  - high_school_microeconomics",
+    "acc,none": 0.7478991596638656,
+    "acc_stderr,none": 0.02820554503327774
+  },
+  "mmlu_high_school_psychology": {
+    "alias": "  - high_school_psychology",
+    "acc,none": 0.8752293577981651,
+    "acc_stderr,none": 0.014168298359156423
+  },
+  "mmlu_human_sexuality": {
+    "alias": "  - human_sexuality",
+    "acc,none": 0.7633587786259542,
+    "acc_stderr,none": 0.03727673575596918
+  },
+  "mmlu_professional_psychology": {
+    "alias": "  - professional_psychology",
+    "acc,none": 0.684640522875817,
+    "acc_stderr,none": 0.01879808628488684
+  },
+  "mmlu_public_relations": {
+    "alias": "  - public_relations",
+    "acc,none": 0.6454545454545455,
+    "acc_stderr,none": 0.04582004841505413
+  },
+  "mmlu_security_studies": {
+    "alias": "  - security_studies",
+    "acc,none": 0.726530612244898,
+    "acc_stderr,none": 0.028535560337128486
+  },
+  "mmlu_sociology": {
+    "alias": "  - sociology",
+    "acc,none": 0.8258706467661692,
+    "acc_stderr,none": 0.026814951200421624
+  },
+  "mmlu_us_foreign_policy": {
+    "alias": "  - us_foreign_policy",
+    "acc,none": 0.87,
+    "acc_stderr,none": 0.033799766898963114
+  },
+  "mmlu_stem": {
+    "acc,none": 0.5493181097367587,
+    "acc_stderr,none": 0.008485562908428945,
+    "alias": " - stem"
+  },
+  "mmlu_abstract_algebra": {
+    "alias": "  - abstract_algebra",
+    "acc,none": 0.35,
+    "acc_stderr,none": 0.04793724854411023
+  },
+  "mmlu_anatomy": {
+    "alias": "  - anatomy",
+    "acc,none": 0.674074074074074,
+    "acc_stderr,none": 0.040491220417025006
+  },
+  "mmlu_astronomy": {
+    "alias": "  - astronomy",
+    "acc,none": 0.7302631578947368,
+    "acc_stderr,none": 0.036117805602848975
+  },
+  "mmlu_college_biology": {
+    "alias": "  - college_biology",
+    "acc,none": 0.7916666666666666,
+    "acc_stderr,none": 0.03396116205845331
+  },
+  "mmlu_college_chemistry": {
+    "alias": "  - college_chemistry",
+    "acc,none": 0.46,
+    "acc_stderr,none": 0.05009082659620332
+  },
+  "mmlu_college_computer_science": {
+    "alias": "  - college_computer_science",
+    "acc,none": 0.43,
+    "acc_stderr,none": 0.049756985195624305
+  },
+  "mmlu_college_mathematics": {
+    "alias": "  - college_mathematics",
+    "acc,none": 0.38,
+    "acc_stderr,none": 0.04878317312145634
+  },
+  "mmlu_college_physics": {
+    "alias": "  - college_physics",
+    "acc,none": 0.4117647058823529,
+    "acc_stderr,none": 0.048971049527263624
+  },
+  "mmlu_computer_security": {
+    "alias": "  - computer_security",
+    "acc,none": 0.74,
+    "acc_stderr,none": 0.0440844002276808
+  },
+  "mmlu_conceptual_physics": {
+    "alias": "  - conceptual_physics",
+    "acc,none": 0.5914893617021276,
+    "acc_stderr,none": 0.03213418026701576
+  },
+  "mmlu_electrical_engineering": {
+    "alias": "  - electrical_engineering",
+    "acc,none": 0.6068965517241379,
+    "acc_stderr,none": 0.04070329013707074
+  },
+  "mmlu_elementary_mathematics": {
+    "alias": "  - elementary_mathematics",
+    "acc,none": 0.46296296296296297,
+    "acc_stderr,none": 0.025680564640056906
+  },
+  "mmlu_high_school_biology": {
+    "alias": "  - high_school_biology",
+    "acc,none": 0.7870967741935484,
+    "acc_stderr,none": 0.023287665127268594
+  },
+  "mmlu_high_school_chemistry": {
+    "alias": "  - high_school_chemistry",
+    "acc,none": 0.5615763546798029,
+    "acc_stderr,none": 0.034912078574865175
+  },
+  "mmlu_high_school_computer_science": {
+    "alias": "  - high_school_computer_science",
+    "acc,none": 0.64,
+    "acc_stderr,none": 0.048241815132442176
+  },
+  "mmlu_high_school_mathematics": {
+    "alias": "  - high_school_mathematics",
+    "acc,none": 0.34444444444444444,
+    "acc_stderr,none": 0.02897264888484431
+  },
+  "mmlu_high_school_physics": {
+    "alias": "  - high_school_physics",
+    "acc,none": 0.37748344370860926,
+    "acc_stderr,none": 0.03958027231121572
+  },
+  "mmlu_high_school_statistics": {
+    "alias": "  - high_school_statistics",
+    "acc,none": 0.5185185185185185,
+    "acc_stderr,none": 0.034076320938540516
+  },
+  "mmlu_machine_learning": {
+    "alias": "  - machine_learning",
+    "acc,none": 0.4642857142857143,
+    "acc_stderr,none": 0.04733667890053756
+  },
+  "openbookqa": {
+    "alias": "openbookqa",
+    "acc,none": 0.378,
+    "acc_stderr,none": 0.02170655082451827,
+    "acc_norm,none": 0.462,
+    "acc_norm_stderr,none": 0.022318338119870523
+  }
+}

original_model/results_['boolq', 'lambada_openai', 'logiqa', 'mmlu', 'openbookqa', 'hellaswag']_raw.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b274b94381616c2afd378f275dbc26d14ed2e102090b2dabde8dddf1987c94bb
+size 131969634

original_model/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
+size 14244

original_model/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c408953b91b7cdca662a1e3770cf93417d3529bf71a509c82bd616f97514b48
+size 1064

original_model/special_tokens_map.json

@@ -13,5 +13,11 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<|begin_of_text|>"
+  "pad_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

original_model/tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:71c1736d713a012aa4720d05a6c32c6992e9d278ea88cedd36008f14fd616116
-size 17210091
+oid sha256:52716f60c3ad328509fa37cdded9a2f1196ecae463f5480f5d38c66a25e7a7dc
+size 17210019

original_model/tokenizer_config.json

@@ -2053,11 +2053,15 @@
   "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|eot_id|>",
+  "max_length": null,
   "model_input_names": [
     "input_ids",
     "attention_mask"
   ],
   "model_max_length": 131072,
+  "pad_to_multiple_of": null,
   "pad_token": "<|begin_of_text|>",
+  "pad_token_type_id": 0,
+  "padding_side": "left",
   "tokenizer_class": "PreTrainedTokenizerFast"
 }

original_model/trainer_state.json

@@ -0,0 +1,2049 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 200,
+  "global_step": 7189,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00013910140492418973,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 9.998608985950758e-06,
+      "loss": 0.8981,
+      "step": 1
+    },
+    {
+      "epoch": 0.0034775351231047433,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 9.965224648768953e-06,
+      "loss": 0.9497,
+      "step": 25
+    },
+    {
+      "epoch": 0.006955070246209487,
+      "grad_norm": 0.1845703125,
+      "learning_rate": 9.930449297537907e-06,
+      "loss": 1.0367,
+      "step": 50
+    },
+    {
+      "epoch": 0.01043260536931423,
+      "grad_norm": 0.134765625,
+      "learning_rate": 9.895673946306859e-06,
+      "loss": 0.9144,
+      "step": 75
+    },
+    {
+      "epoch": 0.013910140492418973,
+      "grad_norm": 0.384765625,
+      "learning_rate": 9.860898595075811e-06,
+      "loss": 0.9554,
+      "step": 100
+    },
+    {
+      "epoch": 0.017387675615523717,
+      "grad_norm": 0.1337890625,
+      "learning_rate": 9.826123243844763e-06,
+      "loss": 0.9696,
+      "step": 125
+    },
+    {
+      "epoch": 0.02086521073862846,
+      "grad_norm": 0.408203125,
+      "learning_rate": 9.791347892613715e-06,
+      "loss": 0.9559,
+      "step": 150
+    },
+    {
+      "epoch": 0.024342745861733205,
+      "grad_norm": 0.78515625,
+      "learning_rate": 9.756572541382668e-06,
+      "loss": 0.9829,
+      "step": 175
+    },
+    {
+      "epoch": 0.027820280984837947,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 9.721797190151621e-06,
+      "loss": 0.9825,
+      "step": 200
+    },
+    {
+      "epoch": 0.03129781610794269,
+      "grad_norm": 0.21484375,
+      "learning_rate": 9.687021838920574e-06,
+      "loss": 0.9892,
+      "step": 225
+    },
+    {
+      "epoch": 0.034775351231047434,
+      "grad_norm": 0.25390625,
+      "learning_rate": 9.652246487689526e-06,
+      "loss": 0.9598,
+      "step": 250
+    },
+    {
+      "epoch": 0.03825288635415218,
+      "grad_norm": 0.140625,
+      "learning_rate": 9.61747113645848e-06,
+      "loss": 0.925,
+      "step": 275
+    },
+    {
+      "epoch": 0.04173042147725692,
+      "grad_norm": 0.255859375,
+      "learning_rate": 9.582695785227432e-06,
+      "loss": 1.0342,
+      "step": 300
+    },
+    {
+      "epoch": 0.045207956600361664,
+      "grad_norm": 0.71875,
+      "learning_rate": 9.547920433996384e-06,
+      "loss": 0.9389,
+      "step": 325
+    },
+    {
+      "epoch": 0.04868549172346641,
+      "grad_norm": 0.376953125,
+      "learning_rate": 9.513145082765338e-06,
+      "loss": 0.9429,
+      "step": 350
+    },
+    {
+      "epoch": 0.05216302684657115,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 9.47836973153429e-06,
+      "loss": 0.9882,
+      "step": 375
+    },
+    {
+      "epoch": 0.05564056196967589,
+      "grad_norm": 0.25,
+      "learning_rate": 9.443594380303242e-06,
+      "loss": 0.9416,
+      "step": 400
+    },
+    {
+      "epoch": 0.05911809709278064,
+      "grad_norm": 0.140625,
+      "learning_rate": 9.408819029072194e-06,
+      "loss": 0.8821,
+      "step": 425
+    },
+    {
+      "epoch": 0.06259563221588538,
+      "grad_norm": 0.203125,
+      "learning_rate": 9.374043677841146e-06,
+      "loss": 0.9094,
+      "step": 450
+    },
+    {
+      "epoch": 0.06607316733899013,
+      "grad_norm": 0.1123046875,
+      "learning_rate": 9.339268326610099e-06,
+      "loss": 0.9274,
+      "step": 475
+    },
+    {
+      "epoch": 0.06955070246209487,
+      "grad_norm": 0.2890625,
+      "learning_rate": 9.304492975379052e-06,
+      "loss": 1.1017,
+      "step": 500
+    },
+    {
+      "epoch": 0.0730282375851996,
+      "grad_norm": 0.296875,
+      "learning_rate": 9.269717624148005e-06,
+      "loss": 0.9424,
+      "step": 525
+    },
+    {
+      "epoch": 0.07650577270830436,
+      "grad_norm": 0.166015625,
+      "learning_rate": 9.234942272916957e-06,
+      "loss": 0.9973,
+      "step": 550
+    },
+    {
+      "epoch": 0.0799833078314091,
+      "grad_norm": 0.140625,
+      "learning_rate": 9.20016692168591e-06,
+      "loss": 0.9476,
+      "step": 575
+    },
+    {
+      "epoch": 0.08346084295451384,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 9.165391570454863e-06,
+      "loss": 0.9932,
+      "step": 600
+    },
+    {
+      "epoch": 0.08693837807761859,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 9.130616219223815e-06,
+      "loss": 0.9203,
+      "step": 625
+    },
+    {
+      "epoch": 0.09041591320072333,
+      "grad_norm": 0.158203125,
+      "learning_rate": 9.095840867992769e-06,
+      "loss": 0.9127,
+      "step": 650
+    },
+    {
+      "epoch": 0.09389344832382807,
+      "grad_norm": 0.14453125,
+      "learning_rate": 9.061065516761721e-06,
+      "loss": 0.8923,
+      "step": 675
+    },
+    {
+      "epoch": 0.09737098344693282,
+      "grad_norm": 1.28125,
+      "learning_rate": 9.026290165530673e-06,
+      "loss": 0.9191,
+      "step": 700
+    },
+    {
+      "epoch": 0.10084851857003756,
+      "grad_norm": 0.298828125,
+      "learning_rate": 8.991514814299625e-06,
+      "loss": 0.9355,
+      "step": 725
+    },
+    {
+      "epoch": 0.1043260536931423,
+      "grad_norm": 0.134765625,
+      "learning_rate": 8.956739463068577e-06,
+      "loss": 0.9215,
+      "step": 750
+    },
+    {
+      "epoch": 0.10780358881624705,
+      "grad_norm": 0.22265625,
+      "learning_rate": 8.92196411183753e-06,
+      "loss": 0.9784,
+      "step": 775
+    },
+    {
+      "epoch": 0.11128112393935179,
+      "grad_norm": 0.1943359375,
+      "learning_rate": 8.887188760606483e-06,
+      "loss": 0.9981,
+      "step": 800
+    },
+    {
+      "epoch": 0.11475865906245653,
+      "grad_norm": 0.203125,
+      "learning_rate": 8.852413409375435e-06,
+      "loss": 1.0032,
+      "step": 825
+    },
+    {
+      "epoch": 0.11823619418556128,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 8.817638058144388e-06,
+      "loss": 0.964,
+      "step": 850
+    },
+    {
+      "epoch": 0.12171372930866602,
+      "grad_norm": 0.1650390625,
+      "learning_rate": 8.782862706913341e-06,
+      "loss": 0.9522,
+      "step": 875
+    },
+    {
+      "epoch": 0.12519126443177075,
+      "grad_norm": 0.1337890625,
+      "learning_rate": 8.748087355682294e-06,
+      "loss": 1.0285,
+      "step": 900
+    },
+    {
+      "epoch": 0.1286687995548755,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 8.713312004451246e-06,
+      "loss": 1.077,
+      "step": 925
+    },
+    {
+      "epoch": 0.13214633467798026,
+      "grad_norm": 0.1376953125,
+      "learning_rate": 8.678536653220198e-06,
+      "loss": 0.9334,
+      "step": 950
+    },
+    {
+      "epoch": 0.13562386980108498,
+      "grad_norm": 0.189453125,
+      "learning_rate": 8.64376130198915e-06,
+      "loss": 0.915,
+      "step": 975
+    },
+    {
+      "epoch": 0.13910140492418974,
+      "grad_norm": 0.1796875,
+      "learning_rate": 8.608985950758102e-06,
+      "loss": 0.9779,
+      "step": 1000
+    },
+    {
+      "epoch": 0.1425789400472945,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 8.574210599527056e-06,
+      "loss": 0.93,
+      "step": 1025
+    },
+    {
+      "epoch": 0.1460564751703992,
+      "grad_norm": 0.375,
+      "learning_rate": 8.539435248296008e-06,
+      "loss": 1.0351,
+      "step": 1050
+    },
+    {
+      "epoch": 0.14953401029350397,
+      "grad_norm": 0.21484375,
+      "learning_rate": 8.50465989706496e-06,
+      "loss": 0.9132,
+      "step": 1075
+    },
+    {
+      "epoch": 0.15301154541660872,
+      "grad_norm": 0.275390625,
+      "learning_rate": 8.469884545833914e-06,
+      "loss": 0.9116,
+      "step": 1100
+    },
+    {
+      "epoch": 0.15648908053971344,
+      "grad_norm": 0.244140625,
+      "learning_rate": 8.435109194602866e-06,
+      "loss": 0.9694,
+      "step": 1125
+    },
+    {
+      "epoch": 0.1599666156628182,
+      "grad_norm": 0.119140625,
+      "learning_rate": 8.400333843371819e-06,
+      "loss": 0.9026,
+      "step": 1150
+    },
+    {
+      "epoch": 0.16344415078592295,
+      "grad_norm": 0.38671875,
+      "learning_rate": 8.365558492140772e-06,
+      "loss": 1.1708,
+      "step": 1175
+    },
+    {
+      "epoch": 0.16692168590902767,
+      "grad_norm": 0.458984375,
+      "learning_rate": 8.330783140909725e-06,
+      "loss": 0.9734,
+      "step": 1200
+    },
+    {
+      "epoch": 0.17039922103213243,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 8.296007789678677e-06,
+      "loss": 1.0151,
+      "step": 1225
+    },
+    {
+      "epoch": 0.17387675615523718,
+      "grad_norm": 0.337890625,
+      "learning_rate": 8.261232438447629e-06,
+      "loss": 0.9712,
+      "step": 1250
+    },
+    {
+      "epoch": 0.1773542912783419,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 8.226457087216581e-06,
+      "loss": 0.9553,
+      "step": 1275
+    },
+    {
+      "epoch": 0.18083182640144665,
+      "grad_norm": 0.1357421875,
+      "learning_rate": 8.191681735985533e-06,
+      "loss": 0.9089,
+      "step": 1300
+    },
+    {
+      "epoch": 0.1843093615245514,
+      "grad_norm": 0.1904296875,
+      "learning_rate": 8.156906384754487e-06,
+      "loss": 0.9928,
+      "step": 1325
+    },
+    {
+      "epoch": 0.18778689664765613,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 8.12213103352344e-06,
+      "loss": 0.9562,
+      "step": 1350
+    },
+    {
+      "epoch": 0.19126443177076088,
+      "grad_norm": 0.259765625,
+      "learning_rate": 8.087355682292391e-06,
+      "loss": 0.963,
+      "step": 1375
+    },
+    {
+      "epoch": 0.19474196689386564,
+      "grad_norm": 0.2890625,
+      "learning_rate": 8.052580331061345e-06,
+      "loss": 1.0053,
+      "step": 1400
+    },
+    {
+      "epoch": 0.19821950201697036,
+      "grad_norm": 0.1640625,
+      "learning_rate": 8.017804979830297e-06,
+      "loss": 0.9583,
+      "step": 1425
+    },
+    {
+      "epoch": 0.2016970371400751,
+      "grad_norm": 0.2734375,
+      "learning_rate": 7.98302962859925e-06,
+      "loss": 0.9633,
+      "step": 1450
+    },
+    {
+      "epoch": 0.20517457226317987,
+      "grad_norm": 0.185546875,
+      "learning_rate": 7.948254277368203e-06,
+      "loss": 0.9325,
+      "step": 1475
+    },
+    {
+      "epoch": 0.2086521073862846,
+      "grad_norm": 0.154296875,
+      "learning_rate": 7.913478926137156e-06,
+      "loss": 0.9567,
+      "step": 1500
+    },
+    {
+      "epoch": 0.21212964250938934,
+      "grad_norm": 0.26171875,
+      "learning_rate": 7.878703574906108e-06,
+      "loss": 0.9544,
+      "step": 1525
+    },
+    {
+      "epoch": 0.2156071776324941,
+      "grad_norm": 0.2138671875,
+      "learning_rate": 7.84392822367506e-06,
+      "loss": 0.8735,
+      "step": 1550
+    },
+    {
+      "epoch": 0.21908471275559882,
+      "grad_norm": 0.26171875,
+      "learning_rate": 7.809152872444012e-06,
+      "loss": 0.9212,
+      "step": 1575
+    },
+    {
+      "epoch": 0.22256224787870357,
+      "grad_norm": 0.1845703125,
+      "learning_rate": 7.774377521212964e-06,
+      "loss": 0.9532,
+      "step": 1600
+    },
+    {
+      "epoch": 0.22603978300180833,
+      "grad_norm": 0.2333984375,
+      "learning_rate": 7.739602169981918e-06,
+      "loss": 0.9295,
+      "step": 1625
+    },
+    {
+      "epoch": 0.22951731812491305,
+      "grad_norm": 0.2734375,
+      "learning_rate": 7.70482681875087e-06,
+      "loss": 1.028,
+      "step": 1650
+    },
+    {
+      "epoch": 0.2329948532480178,
+      "grad_norm": 0.1201171875,
+      "learning_rate": 7.670051467519822e-06,
+      "loss": 0.8971,
+      "step": 1675
+    },
+    {
+      "epoch": 0.23647238837112255,
+      "grad_norm": 0.1376953125,
+      "learning_rate": 7.635276116288776e-06,
+      "loss": 0.9231,
+      "step": 1700
+    },
+    {
+      "epoch": 0.23994992349422728,
+      "grad_norm": 0.15234375,
+      "learning_rate": 7.6005007650577274e-06,
+      "loss": 0.9017,
+      "step": 1725
+    },
+    {
+      "epoch": 0.24342745861733203,
+      "grad_norm": 0.193359375,
+      "learning_rate": 7.56572541382668e-06,
+      "loss": 0.9792,
+      "step": 1750
+    },
+    {
+      "epoch": 0.24690499374043678,
+      "grad_norm": 0.2451171875,
+      "learning_rate": 7.5309500625956334e-06,
+      "loss": 0.9671,
+      "step": 1775
+    },
+    {
+      "epoch": 0.2503825288635415,
+      "grad_norm": 0.177734375,
+      "learning_rate": 7.496174711364586e-06,
+      "loss": 0.8667,
+      "step": 1800
+    },
+    {
+      "epoch": 0.2538600639866463,
+      "grad_norm": 0.181640625,
+      "learning_rate": 7.461399360133538e-06,
+      "loss": 0.9687,
+      "step": 1825
+    },
+    {
+      "epoch": 0.257337599109751,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 7.426624008902491e-06,
+      "loss": 0.875,
+      "step": 1850
+    },
+    {
+      "epoch": 0.26081513423285574,
+      "grad_norm": 0.224609375,
+      "learning_rate": 7.391848657671443e-06,
+      "loss": 0.8993,
+      "step": 1875
+    },
+    {
+      "epoch": 0.2642926693559605,
+      "grad_norm": 0.25,
+      "learning_rate": 7.357073306440395e-06,
+      "loss": 0.8771,
+      "step": 1900
+    },
+    {
+      "epoch": 0.26777020447906524,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 7.322297955209349e-06,
+      "loss": 0.9736,
+      "step": 1925
+    },
+    {
+      "epoch": 0.27124773960216997,
+      "grad_norm": 0.1904296875,
+      "learning_rate": 7.287522603978301e-06,
+      "loss": 0.9709,
+      "step": 1950
+    },
+    {
+      "epoch": 0.27472527472527475,
+      "grad_norm": 0.2197265625,
+      "learning_rate": 7.252747252747253e-06,
+      "loss": 0.9675,
+      "step": 1975
+    },
+    {
+      "epoch": 0.2782028098483795,
+      "grad_norm": 0.21875,
+      "learning_rate": 7.217971901516206e-06,
+      "loss": 0.9274,
+      "step": 2000
+    },
+    {
+      "epoch": 0.2816803449714842,
+      "grad_norm": 0.83203125,
+      "learning_rate": 7.183196550285158e-06,
+      "loss": 1.0012,
+      "step": 2025
+    },
+    {
+      "epoch": 0.285157880094589,
+      "grad_norm": 0.240234375,
+      "learning_rate": 7.1484211990541105e-06,
+      "loss": 0.9154,
+      "step": 2050
+    },
+    {
+      "epoch": 0.2886354152176937,
+      "grad_norm": 0.3046875,
+      "learning_rate": 7.113645847823064e-06,
+      "loss": 0.9657,
+      "step": 2075
+    },
+    {
+      "epoch": 0.2921129503407984,
+      "grad_norm": 0.2109375,
+      "learning_rate": 7.0788704965920165e-06,
+      "loss": 0.9909,
+      "step": 2100
+    },
+    {
+      "epoch": 0.2955904854639032,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 7.044095145360969e-06,
+      "loss": 0.9059,
+      "step": 2125
+    },
+    {
+      "epoch": 0.29906802058700793,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 7.009319794129922e-06,
+      "loss": 0.9988,
+      "step": 2150
+    },
+    {
+      "epoch": 0.30254555571011266,
+      "grad_norm": 0.33203125,
+      "learning_rate": 6.974544442898874e-06,
+      "loss": 0.9623,
+      "step": 2175
+    },
+    {
+      "epoch": 0.30602309083321744,
+      "grad_norm": 0.1884765625,
+      "learning_rate": 6.939769091667826e-06,
+      "loss": 0.9208,
+      "step": 2200
+    },
+    {
+      "epoch": 0.30950062595632216,
+      "grad_norm": 0.318359375,
+      "learning_rate": 6.90499374043678e-06,
+      "loss": 0.9536,
+      "step": 2225
+    },
+    {
+      "epoch": 0.3129781610794269,
+      "grad_norm": 0.10888671875,
+      "learning_rate": 6.870218389205732e-06,
+      "loss": 1.0221,
+      "step": 2250
+    },
+    {
+      "epoch": 0.31645569620253167,
+      "grad_norm": 0.166015625,
+      "learning_rate": 6.835443037974684e-06,
+      "loss": 1.0849,
+      "step": 2275
+    },
+    {
+      "epoch": 0.3199332313256364,
+      "grad_norm": 0.259765625,
+      "learning_rate": 6.800667686743637e-06,
+      "loss": 0.9612,
+      "step": 2300
+    },
+    {
+      "epoch": 0.3234107664487411,
+      "grad_norm": 0.10546875,
+      "learning_rate": 6.765892335512589e-06,
+      "loss": 0.977,
+      "step": 2325
+    },
+    {
+      "epoch": 0.3268883015718459,
+      "grad_norm": 0.51171875,
+      "learning_rate": 6.7311169842815415e-06,
+      "loss": 0.9291,
+      "step": 2350
+    },
+    {
+      "epoch": 0.3303658366949506,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 6.6963416330504945e-06,
+      "loss": 0.9471,
+      "step": 2375
+    },
+    {
+      "epoch": 0.33384337181805535,
+      "grad_norm": 0.609375,
+      "learning_rate": 6.6615662818194475e-06,
+      "loss": 0.8984,
+      "step": 2400
+    },
+    {
+      "epoch": 0.3373209069411601,
+      "grad_norm": 0.19921875,
+      "learning_rate": 6.6267909305884e-06,
+      "loss": 1.0189,
+      "step": 2425
+    },
+    {
+      "epoch": 0.34079844206426485,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 6.592015579357353e-06,
+      "loss": 0.915,
+      "step": 2450
+    },
+    {
+      "epoch": 0.3442759771873696,
+      "grad_norm": 0.32421875,
+      "learning_rate": 6.557240228126305e-06,
+      "loss": 0.9904,
+      "step": 2475
+    },
+    {
+      "epoch": 0.34775351231047436,
+      "grad_norm": 0.35546875,
+      "learning_rate": 6.522464876895257e-06,
+      "loss": 1.0573,
+      "step": 2500
+    },
+    {
+      "epoch": 0.3512310474335791,
+      "grad_norm": 0.3671875,
+      "learning_rate": 6.48768952566421e-06,
+      "loss": 1.009,
+      "step": 2525
+    },
+    {
+      "epoch": 0.3547085825566838,
+      "grad_norm": 0.337890625,
+      "learning_rate": 6.452914174433162e-06,
+      "loss": 0.9061,
+      "step": 2550
+    },
+    {
+      "epoch": 0.3581861176797886,
+      "grad_norm": 0.19921875,
+      "learning_rate": 6.418138823202114e-06,
+      "loss": 0.8657,
+      "step": 2575
+    },
+    {
+      "epoch": 0.3616636528028933,
+      "grad_norm": 0.201171875,
+      "learning_rate": 6.383363471971068e-06,
+      "loss": 0.9302,
+      "step": 2600
+    },
+    {
+      "epoch": 0.36514118792599803,
+      "grad_norm": 0.2421875,
+      "learning_rate": 6.34858812074002e-06,
+      "loss": 0.9776,
+      "step": 2625
+    },
+    {
+      "epoch": 0.3686187230491028,
+      "grad_norm": 0.2177734375,
+      "learning_rate": 6.313812769508972e-06,
+      "loss": 0.9165,
+      "step": 2650
+    },
+    {
+      "epoch": 0.37209625817220754,
+      "grad_norm": 0.162109375,
+      "learning_rate": 6.2790374182779246e-06,
+      "loss": 0.9424,
+      "step": 2675
+    },
+    {
+      "epoch": 0.37557379329531226,
+      "grad_norm": 0.171875,
+      "learning_rate": 6.2442620670468776e-06,
+      "loss": 0.9479,
+      "step": 2700
+    },
+    {
+      "epoch": 0.37905132841841704,
+      "grad_norm": 0.1640625,
+      "learning_rate": 6.20948671581583e-06,
+      "loss": 1.0845,
+      "step": 2725
+    },
+    {
+      "epoch": 0.38252886354152177,
+      "grad_norm": 0.1953125,
+      "learning_rate": 6.174711364584782e-06,
+      "loss": 0.9801,
+      "step": 2750
+    },
+    {
+      "epoch": 0.3860063986646265,
+      "grad_norm": 0.1640625,
+      "learning_rate": 6.139936013353736e-06,
+      "loss": 0.8536,
+      "step": 2775
+    },
+    {
+      "epoch": 0.3894839337877313,
+      "grad_norm": 0.400390625,
+      "learning_rate": 6.105160662122688e-06,
+      "loss": 0.9453,
+      "step": 2800
+    },
+    {
+      "epoch": 0.392961468910836,
+      "grad_norm": 1.6796875,
+      "learning_rate": 6.07038531089164e-06,
+      "loss": 1.0559,
+      "step": 2825
+    },
+    {
+      "epoch": 0.3964390040339407,
+      "grad_norm": 0.236328125,
+      "learning_rate": 6.035609959660593e-06,
+      "loss": 0.9228,
+      "step": 2850
+    },
+    {
+      "epoch": 0.3999165391570455,
+      "grad_norm": 0.447265625,
+      "learning_rate": 6.000834608429545e-06,
+      "loss": 0.8822,
+      "step": 2875
+    },
+    {
+      "epoch": 0.4033940742801502,
+      "grad_norm": 0.3046875,
+      "learning_rate": 5.966059257198497e-06,
+      "loss": 0.9643,
+      "step": 2900
+    },
+    {
+      "epoch": 0.40687160940325495,
+      "grad_norm": 0.1962890625,
+      "learning_rate": 5.931283905967451e-06,
+      "loss": 0.9333,
+      "step": 2925
+    },
+    {
+      "epoch": 0.41034914452635973,
+      "grad_norm": 0.197265625,
+      "learning_rate": 5.896508554736403e-06,
+      "loss": 0.8989,
+      "step": 2950
+    },
+    {
+      "epoch": 0.41382667964946446,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 5.8617332035053555e-06,
+      "loss": 0.9434,
+      "step": 2975
+    },
+    {
+      "epoch": 0.4173042147725692,
+      "grad_norm": 0.380859375,
+      "learning_rate": 5.8269578522743085e-06,
+      "loss": 0.8988,
+      "step": 3000
+    },
+    {
+      "epoch": 0.42078174989567396,
+      "grad_norm": 0.205078125,
+      "learning_rate": 5.792182501043261e-06,
+      "loss": 0.973,
+      "step": 3025
+    },
+    {
+      "epoch": 0.4242592850187787,
+      "grad_norm": 0.294921875,
+      "learning_rate": 5.757407149812213e-06,
+      "loss": 0.96,
+      "step": 3050
+    },
+    {
+      "epoch": 0.4277368201418834,
+      "grad_norm": 0.494140625,
+      "learning_rate": 5.722631798581167e-06,
+      "loss": 0.9964,
+      "step": 3075
+    },
+    {
+      "epoch": 0.4312143552649882,
+      "grad_norm": 0.11279296875,
+      "learning_rate": 5.687856447350119e-06,
+      "loss": 0.9466,
+      "step": 3100
+    },
+    {
+      "epoch": 0.4346918903880929,
+      "grad_norm": 0.1943359375,
+      "learning_rate": 5.653081096119071e-06,
+      "loss": 0.9676,
+      "step": 3125
+    },
+    {
+      "epoch": 0.43816942551119764,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 5.618305744888024e-06,
+      "loss": 0.9048,
+      "step": 3150
+    },
+    {
+      "epoch": 0.4416469606343024,
+      "grad_norm": 0.1884765625,
+      "learning_rate": 5.583530393656976e-06,
+      "loss": 0.9142,
+      "step": 3175
+    },
+    {
+      "epoch": 0.44512449575740715,
+      "grad_norm": 0.2255859375,
+      "learning_rate": 5.548755042425928e-06,
+      "loss": 0.9951,
+      "step": 3200
+    },
+    {
+      "epoch": 0.44860203088051187,
+      "grad_norm": 0.1416015625,
+      "learning_rate": 5.513979691194882e-06,
+      "loss": 1.1234,
+      "step": 3225
+    },
+    {
+      "epoch": 0.45207956600361665,
+      "grad_norm": 0.404296875,
+      "learning_rate": 5.479204339963834e-06,
+      "loss": 0.9482,
+      "step": 3250
+    },
+    {
+      "epoch": 0.4555571011267214,
+      "grad_norm": 0.302734375,
+      "learning_rate": 5.4444289887327865e-06,
+      "loss": 1.1495,
+      "step": 3275
+    },
+    {
+      "epoch": 0.4590346362498261,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 5.4096536375017395e-06,
+      "loss": 0.9134,
+      "step": 3300
+    },
+    {
+      "epoch": 0.4625121713729309,
+      "grad_norm": 0.193359375,
+      "learning_rate": 5.374878286270692e-06,
+      "loss": 0.9914,
+      "step": 3325
+    },
+    {
+      "epoch": 0.4659897064960356,
+      "grad_norm": 0.1640625,
+      "learning_rate": 5.340102935039644e-06,
+      "loss": 0.933,
+      "step": 3350
+    },
+    {
+      "epoch": 0.46946724161914033,
+      "grad_norm": 0.498046875,
+      "learning_rate": 5.305327583808597e-06,
+      "loss": 1.0268,
+      "step": 3375
+    },
+    {
+      "epoch": 0.4729447767422451,
+      "grad_norm": 0.2353515625,
+      "learning_rate": 5.270552232577549e-06,
+      "loss": 0.9866,
+      "step": 3400
+    },
+    {
+      "epoch": 0.47642231186534983,
+      "grad_norm": 0.294921875,
+      "learning_rate": 5.235776881346502e-06,
+      "loss": 0.9109,
+      "step": 3425
+    },
+    {
+      "epoch": 0.47989984698845456,
+      "grad_norm": 0.11474609375,
+      "learning_rate": 5.201001530115455e-06,
+      "loss": 0.911,
+      "step": 3450
+    },
+    {
+      "epoch": 0.48337738211155934,
+      "grad_norm": 0.30859375,
+      "learning_rate": 5.166226178884407e-06,
+      "loss": 1.2343,
+      "step": 3475
+    },
+    {
+      "epoch": 0.48685491723466406,
+      "grad_norm": 0.15234375,
+      "learning_rate": 5.131450827653359e-06,
+      "loss": 1.028,
+      "step": 3500
+    },
+    {
+      "epoch": 0.4903324523577688,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 5.096675476422312e-06,
+      "loss": 0.9523,
+      "step": 3525
+    },
+    {
+      "epoch": 0.49380998748087357,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 5.061900125191264e-06,
+      "loss": 0.9315,
+      "step": 3550
+    },
+    {
+      "epoch": 0.4972875226039783,
+      "grad_norm": 0.220703125,
+      "learning_rate": 5.0271247739602165e-06,
+      "loss": 0.9098,
+      "step": 3575
+    },
+    {
+      "epoch": 0.500765057727083,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 4.99234942272917e-06,
+      "loss": 0.9854,
+      "step": 3600
+    },
+    {
+      "epoch": 0.5042425928501878,
+      "grad_norm": 0.1484375,
+      "learning_rate": 4.9575740714981225e-06,
+      "loss": 0.9635,
+      "step": 3625
+    },
+    {
+      "epoch": 0.5077201279732926,
+      "grad_norm": 0.1923828125,
+      "learning_rate": 4.9227987202670755e-06,
+      "loss": 0.9722,
+      "step": 3650
+    },
+    {
+      "epoch": 0.5111976630963972,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 4.888023369036028e-06,
+      "loss": 0.9458,
+      "step": 3675
+    },
+    {
+      "epoch": 0.514675198219502,
+      "grad_norm": 0.216796875,
+      "learning_rate": 4.85324801780498e-06,
+      "loss": 0.949,
+      "step": 3700
+    },
+    {
+      "epoch": 0.5181527333426068,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 4.818472666573933e-06,
+      "loss": 0.9858,
+      "step": 3725
+    },
+    {
+      "epoch": 0.5216302684657115,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 4.783697315342886e-06,
+      "loss": 0.9366,
+      "step": 3750
+    },
+    {
+      "epoch": 0.5251078035888163,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 4.748921964111838e-06,
+      "loss": 0.9136,
+      "step": 3775
+    },
+    {
+      "epoch": 0.528585338711921,
+      "grad_norm": 0.255859375,
+      "learning_rate": 4.71414661288079e-06,
+      "loss": 0.8959,
+      "step": 3800
+    },
+    {
+      "epoch": 0.5320628738350257,
+      "grad_norm": 0.2890625,
+      "learning_rate": 4.679371261649743e-06,
+      "loss": 1.0477,
+      "step": 3825
+    },
+    {
+      "epoch": 0.5355404089581305,
+      "grad_norm": 1.203125,
+      "learning_rate": 4.644595910418695e-06,
+      "loss": 0.9659,
+      "step": 3850
+    },
+    {
+      "epoch": 0.5390179440812353,
+      "grad_norm": 0.546875,
+      "learning_rate": 4.609820559187648e-06,
+      "loss": 0.9351,
+      "step": 3875
+    },
+    {
+      "epoch": 0.5424954792043399,
+      "grad_norm": 0.466796875,
+      "learning_rate": 4.575045207956601e-06,
+      "loss": 0.9646,
+      "step": 3900
+    },
+    {
+      "epoch": 0.5459730143274447,
+      "grad_norm": 0.173828125,
+      "learning_rate": 4.5402698567255535e-06,
+      "loss": 1.0591,
+      "step": 3925
+    },
+    {
+      "epoch": 0.5494505494505495,
+      "grad_norm": 0.2353515625,
+      "learning_rate": 4.505494505494506e-06,
+      "loss": 0.9134,
+      "step": 3950
+    },
+    {
+      "epoch": 0.5529280845736542,
+      "grad_norm": 0.181640625,
+      "learning_rate": 4.470719154263458e-06,
+      "loss": 0.9683,
+      "step": 3975
+    },
+    {
+      "epoch": 0.556405619696759,
+      "grad_norm": 0.33203125,
+      "learning_rate": 4.435943803032411e-06,
+      "loss": 0.9613,
+      "step": 4000
+    },
+    {
+      "epoch": 0.5598831548198637,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 4.401168451801364e-06,
+      "loss": 0.9213,
+      "step": 4025
+    },
+    {
+      "epoch": 0.5633606899429684,
+      "grad_norm": 2.296875,
+      "learning_rate": 4.366393100570316e-06,
+      "loss": 0.8767,
+      "step": 4050
+    },
+    {
+      "epoch": 0.5668382250660732,
+      "grad_norm": 0.1650390625,
+      "learning_rate": 4.331617749339269e-06,
+      "loss": 0.9816,
+      "step": 4075
+    },
+    {
+      "epoch": 0.570315760189178,
+      "grad_norm": 0.185546875,
+      "learning_rate": 4.296842398108221e-06,
+      "loss": 0.9544,
+      "step": 4100
+    },
+    {
+      "epoch": 0.5737932953122826,
+      "grad_norm": 0.3203125,
+      "learning_rate": 4.262067046877173e-06,
+      "loss": 0.9496,
+      "step": 4125
+    },
+    {
+      "epoch": 0.5772708304353874,
+      "grad_norm": 0.296875,
+      "learning_rate": 4.227291695646126e-06,
+      "loss": 0.9774,
+      "step": 4150
+    },
+    {
+      "epoch": 0.5807483655584922,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 4.192516344415079e-06,
+      "loss": 0.9412,
+      "step": 4175
+    },
+    {
+      "epoch": 0.5842259006815969,
+      "grad_norm": 0.1552734375,
+      "learning_rate": 4.157740993184031e-06,
+      "loss": 0.9162,
+      "step": 4200
+    },
+    {
+      "epoch": 0.5877034358047016,
+      "grad_norm": 0.296875,
+      "learning_rate": 4.122965641952984e-06,
+      "loss": 0.9275,
+      "step": 4225
+    },
+    {
+      "epoch": 0.5911809709278064,
+      "grad_norm": 0.20703125,
+      "learning_rate": 4.088190290721937e-06,
+      "loss": 0.9903,
+      "step": 4250
+    },
+    {
+      "epoch": 0.5946585060509111,
+      "grad_norm": 0.333984375,
+      "learning_rate": 4.053414939490889e-06,
+      "loss": 0.9672,
+      "step": 4275
+    },
+    {
+      "epoch": 0.5981360411740159,
+      "grad_norm": 0.162109375,
+      "learning_rate": 4.018639588259842e-06,
+      "loss": 0.9424,
+      "step": 4300
+    },
+    {
+      "epoch": 0.6016135762971206,
+      "grad_norm": 0.185546875,
+      "learning_rate": 3.983864237028795e-06,
+      "loss": 0.9942,
+      "step": 4325
+    },
+    {
+      "epoch": 0.6050911114202253,
+      "grad_norm": 0.21484375,
+      "learning_rate": 3.949088885797747e-06,
+      "loss": 1.0713,
+      "step": 4350
+    },
+    {
+      "epoch": 0.6085686465433301,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 3.914313534566699e-06,
+      "loss": 0.9087,
+      "step": 4375
+    },
+    {
+      "epoch": 0.6120461816664349,
+      "grad_norm": 0.11962890625,
+      "learning_rate": 3.879538183335652e-06,
+      "loss": 0.9249,
+      "step": 4400
+    },
+    {
+      "epoch": 0.6155237167895395,
+      "grad_norm": 0.9453125,
+      "learning_rate": 3.844762832104604e-06,
+      "loss": 0.9453,
+      "step": 4425
+    },
+    {
+      "epoch": 0.6190012519126443,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 3.809987480873557e-06,
+      "loss": 0.9612,
+      "step": 4450
+    },
+    {
+      "epoch": 0.6224787870357491,
+      "grad_norm": 0.25,
+      "learning_rate": 3.7752121296425098e-06,
+      "loss": 0.9749,
+      "step": 4475
+    },
+    {
+      "epoch": 0.6259563221588538,
+      "grad_norm": 0.162109375,
+      "learning_rate": 3.740436778411462e-06,
+      "loss": 0.9151,
+      "step": 4500
+    },
+    {
+      "epoch": 0.6294338572819586,
+      "grad_norm": 0.1435546875,
+      "learning_rate": 3.705661427180415e-06,
+      "loss": 0.9617,
+      "step": 4525
+    },
+    {
+      "epoch": 0.6329113924050633,
+      "grad_norm": 0.2294921875,
+      "learning_rate": 3.6708860759493675e-06,
+      "loss": 1.1337,
+      "step": 4550
+    },
+    {
+      "epoch": 0.636388927528168,
+      "grad_norm": 0.2275390625,
+      "learning_rate": 3.6361107247183197e-06,
+      "loss": 1.0141,
+      "step": 4575
+    },
+    {
+      "epoch": 0.6398664626512728,
+      "grad_norm": 0.1376953125,
+      "learning_rate": 3.6013353734872727e-06,
+      "loss": 0.8826,
+      "step": 4600
+    },
+    {
+      "epoch": 0.6433439977743776,
+      "grad_norm": 0.2021484375,
+      "learning_rate": 3.5665600222562253e-06,
+      "loss": 0.9162,
+      "step": 4625
+    },
+    {
+      "epoch": 0.6468215328974822,
+      "grad_norm": 0.416015625,
+      "learning_rate": 3.5317846710251774e-06,
+      "loss": 1.1372,
+      "step": 4650
+    },
+    {
+      "epoch": 0.650299068020587,
+      "grad_norm": 0.96875,
+      "learning_rate": 3.49700931979413e-06,
+      "loss": 0.952,
+      "step": 4675
+    },
+    {
+      "epoch": 0.6537766031436918,
+      "grad_norm": 0.296875,
+      "learning_rate": 3.462233968563083e-06,
+      "loss": 0.9671,
+      "step": 4700
+    },
+    {
+      "epoch": 0.6572541382667965,
+      "grad_norm": 0.92578125,
+      "learning_rate": 3.427458617332035e-06,
+      "loss": 1.055,
+      "step": 4725
+    },
+    {
+      "epoch": 0.6607316733899012,
+      "grad_norm": 3.8125,
+      "learning_rate": 3.3926832661009877e-06,
+      "loss": 0.8707,
+      "step": 4750
+    },
+    {
+      "epoch": 0.664209208513006,
+      "grad_norm": 0.341796875,
+      "learning_rate": 3.3579079148699407e-06,
+      "loss": 0.9243,
+      "step": 4775
+    },
+    {
+      "epoch": 0.6676867436361107,
+      "grad_norm": 0.232421875,
+      "learning_rate": 3.323132563638893e-06,
+      "loss": 1.0359,
+      "step": 4800
+    },
+    {
+      "epoch": 0.6711642787592155,
+      "grad_norm": 0.15234375,
+      "learning_rate": 3.2883572124078455e-06,
+      "loss": 0.9355,
+      "step": 4825
+    },
+    {
+      "epoch": 0.6746418138823203,
+      "grad_norm": 0.318359375,
+      "learning_rate": 3.2535818611767985e-06,
+      "loss": 0.9119,
+      "step": 4850
+    },
+    {
+      "epoch": 0.6781193490054249,
+      "grad_norm": 0.1845703125,
+      "learning_rate": 3.2188065099457506e-06,
+      "loss": 0.9348,
+      "step": 4875
+    },
+    {
+      "epoch": 0.6815968841285297,
+      "grad_norm": 0.349609375,
+      "learning_rate": 3.184031158714703e-06,
+      "loss": 1.1575,
+      "step": 4900
+    },
+    {
+      "epoch": 0.6850744192516345,
+      "grad_norm": 0.296875,
+      "learning_rate": 3.149255807483656e-06,
+      "loss": 1.0243,
+      "step": 4925
+    },
+    {
+      "epoch": 0.6885519543747392,
+      "grad_norm": 0.322265625,
+      "learning_rate": 3.1144804562526083e-06,
+      "loss": 0.8819,
+      "step": 4950
+    },
+    {
+      "epoch": 0.6920294894978439,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 3.079705105021561e-06,
+      "loss": 0.931,
+      "step": 4975
+    },
+    {
+      "epoch": 0.6955070246209487,
+      "grad_norm": 0.2451171875,
+      "learning_rate": 3.0449297537905135e-06,
+      "loss": 1.1218,
+      "step": 5000
+    },
+    {
+      "epoch": 0.6989845597440534,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 3.010154402559466e-06,
+      "loss": 0.9186,
+      "step": 5025
+    },
+    {
+      "epoch": 0.7024620948671582,
+      "grad_norm": 0.40625,
+      "learning_rate": 2.9753790513284187e-06,
+      "loss": 1.0308,
+      "step": 5050
+    },
+    {
+      "epoch": 0.7059396299902629,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.9406037000973712e-06,
+      "loss": 1.0625,
+      "step": 5075
+    },
+    {
+      "epoch": 0.7094171651133676,
+      "grad_norm": 0.2412109375,
+      "learning_rate": 2.9058283488663234e-06,
+      "loss": 0.916,
+      "step": 5100
+    },
+    {
+      "epoch": 0.7128947002364724,
+      "grad_norm": 0.1796875,
+      "learning_rate": 2.8710529976352764e-06,
+      "loss": 1.076,
+      "step": 5125
+    },
+    {
+      "epoch": 0.7163722353595772,
+      "grad_norm": 0.23828125,
+      "learning_rate": 2.836277646404229e-06,
+      "loss": 0.9293,
+      "step": 5150
+    },
+    {
+      "epoch": 0.7198497704826818,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 2.801502295173181e-06,
+      "loss": 0.9192,
+      "step": 5175
+    },
+    {
+      "epoch": 0.7233273056057866,
+      "grad_norm": 5.21875,
+      "learning_rate": 2.766726943942134e-06,
+      "loss": 1.1218,
+      "step": 5200
+    },
+    {
+      "epoch": 0.7268048407288914,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 2.7319515927110867e-06,
+      "loss": 0.9646,
+      "step": 5225
+    },
+    {
+      "epoch": 0.7302823758519961,
+      "grad_norm": 0.134765625,
+      "learning_rate": 2.697176241480039e-06,
+      "loss": 1.0199,
+      "step": 5250
+    },
+    {
+      "epoch": 0.7337599109751008,
+      "grad_norm": 0.2578125,
+      "learning_rate": 2.662400890248992e-06,
+      "loss": 0.9171,
+      "step": 5275
+    },
+    {
+      "epoch": 0.7372374460982056,
+      "grad_norm": 0.13671875,
+      "learning_rate": 2.6276255390179444e-06,
+      "loss": 0.9844,
+      "step": 5300
+    },
+    {
+      "epoch": 0.7407149812213103,
+      "grad_norm": 0.203125,
+      "learning_rate": 2.5928501877868966e-06,
+      "loss": 0.9536,
+      "step": 5325
+    },
+    {
+      "epoch": 0.7441925163444151,
+      "grad_norm": 0.26953125,
+      "learning_rate": 2.5580748365558496e-06,
+      "loss": 0.9099,
+      "step": 5350
+    },
+    {
+      "epoch": 0.7476700514675199,
+      "grad_norm": 0.37890625,
+      "learning_rate": 2.523299485324802e-06,
+      "loss": 1.0949,
+      "step": 5375
+    },
+    {
+      "epoch": 0.7511475865906245,
+      "grad_norm": 0.279296875,
+      "learning_rate": 2.4885241340937548e-06,
+      "loss": 0.9524,
+      "step": 5400
+    },
+    {
+      "epoch": 0.7546251217137293,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 2.4537487828627073e-06,
+      "loss": 1.0137,
+      "step": 5425
+    },
+    {
+      "epoch": 0.7581026568368341,
+      "grad_norm": 0.498046875,
+      "learning_rate": 2.4189734316316595e-06,
+      "loss": 1.1407,
+      "step": 5450
+    },
+    {
+      "epoch": 0.7615801919599388,
+      "grad_norm": 0.20703125,
+      "learning_rate": 2.3841980804006125e-06,
+      "loss": 0.9067,
+      "step": 5475
+    },
+    {
+      "epoch": 0.7650577270830435,
+      "grad_norm": 0.26171875,
+      "learning_rate": 2.3494227291695646e-06,
+      "loss": 0.9254,
+      "step": 5500
+    },
+    {
+      "epoch": 0.7685352622061483,
+      "grad_norm": 0.2158203125,
+      "learning_rate": 2.3146473779385172e-06,
+      "loss": 0.9871,
+      "step": 5525
+    },
+    {
+      "epoch": 0.772012797329253,
+      "grad_norm": 0.32421875,
+      "learning_rate": 2.2798720267074702e-06,
+      "loss": 0.9586,
+      "step": 5550
+    },
+    {
+      "epoch": 0.7754903324523578,
+      "grad_norm": 0.341796875,
+      "learning_rate": 2.2450966754764224e-06,
+      "loss": 0.9724,
+      "step": 5575
+    },
+    {
+      "epoch": 0.7789678675754625,
+      "grad_norm": 0.28515625,
+      "learning_rate": 2.210321324245375e-06,
+      "loss": 0.9269,
+      "step": 5600
+    },
+    {
+      "epoch": 0.7824454026985672,
+      "grad_norm": 0.150390625,
+      "learning_rate": 2.1755459730143275e-06,
+      "loss": 0.9433,
+      "step": 5625
+    },
+    {
+      "epoch": 0.785922937821672,
+      "grad_norm": 0.345703125,
+      "learning_rate": 2.14077062178328e-06,
+      "loss": 0.867,
+      "step": 5650
+    },
+    {
+      "epoch": 0.7894004729447768,
+      "grad_norm": 0.1591796875,
+      "learning_rate": 2.1059952705522327e-06,
+      "loss": 0.8577,
+      "step": 5675
+    },
+    {
+      "epoch": 0.7928780080678814,
+      "grad_norm": 0.28125,
+      "learning_rate": 2.0712199193211853e-06,
+      "loss": 1.0268,
+      "step": 5700
+    },
+    {
+      "epoch": 0.7963555431909862,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 2.036444568090138e-06,
+      "loss": 1.0669,
+      "step": 5725
+    },
+    {
+      "epoch": 0.799833078314091,
+      "grad_norm": 0.21484375,
+      "learning_rate": 2.0016692168590904e-06,
+      "loss": 0.9864,
+      "step": 5750
+    },
+    {
+      "epoch": 0.8033106134371957,
+      "grad_norm": 0.234375,
+      "learning_rate": 1.966893865628043e-06,
+      "loss": 0.9397,
+      "step": 5775
+    },
+    {
+      "epoch": 0.8067881485603005,
+      "grad_norm": 0.185546875,
+      "learning_rate": 1.9321185143969956e-06,
+      "loss": 0.8989,
+      "step": 5800
+    },
+    {
+      "epoch": 0.8102656836834052,
+      "grad_norm": 0.193359375,
+      "learning_rate": 1.8973431631659482e-06,
+      "loss": 0.9185,
+      "step": 5825
+    },
+    {
+      "epoch": 0.8137432188065099,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 1.8625678119349007e-06,
+      "loss": 0.9832,
+      "step": 5850
+    },
+    {
+      "epoch": 0.8172207539296147,
+      "grad_norm": 0.51953125,
+      "learning_rate": 1.8277924607038533e-06,
+      "loss": 0.9346,
+      "step": 5875
+    },
+    {
+      "epoch": 0.8206982890527195,
+      "grad_norm": 0.2255859375,
+      "learning_rate": 1.7930171094728057e-06,
+      "loss": 0.9447,
+      "step": 5900
+    },
+    {
+      "epoch": 0.8241758241758241,
+      "grad_norm": 0.12890625,
+      "learning_rate": 1.7582417582417585e-06,
+      "loss": 1.0909,
+      "step": 5925
+    },
+    {
+      "epoch": 0.8276533592989289,
+      "grad_norm": 0.5859375,
+      "learning_rate": 1.723466407010711e-06,
+      "loss": 0.9239,
+      "step": 5950
+    },
+    {
+      "epoch": 0.8311308944220337,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 1.6886910557796634e-06,
+      "loss": 1.0372,
+      "step": 5975
+    },
+    {
+      "epoch": 0.8346084295451384,
+      "grad_norm": 0.1953125,
+      "learning_rate": 1.653915704548616e-06,
+      "loss": 0.9608,
+      "step": 6000
+    },
+    {
+      "epoch": 0.8380859646682431,
+      "grad_norm": 0.27734375,
+      "learning_rate": 1.6191403533175688e-06,
+      "loss": 1.0171,
+      "step": 6025
+    },
+    {
+      "epoch": 0.8415634997913479,
+      "grad_norm": 1.75,
+      "learning_rate": 1.5843650020865212e-06,
+      "loss": 0.9034,
+      "step": 6050
+    },
+    {
+      "epoch": 0.8450410349144526,
+      "grad_norm": 0.458984375,
+      "learning_rate": 1.5495896508554737e-06,
+      "loss": 0.9747,
+      "step": 6075
+    },
+    {
+      "epoch": 0.8485185700375574,
+      "grad_norm": 0.203125,
+      "learning_rate": 1.5148142996244263e-06,
+      "loss": 0.968,
+      "step": 6100
+    },
+    {
+      "epoch": 0.8519961051606622,
+      "grad_norm": 0.23046875,
+      "learning_rate": 1.4800389483933789e-06,
+      "loss": 0.9446,
+      "step": 6125
+    },
+    {
+      "epoch": 0.8554736402837668,
+      "grad_norm": 0.359375,
+      "learning_rate": 1.4452635971623313e-06,
+      "loss": 0.9708,
+      "step": 6150
+    },
+    {
+      "epoch": 0.8589511754068716,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 1.410488245931284e-06,
+      "loss": 0.9382,
+      "step": 6175
+    },
+    {
+      "epoch": 0.8624287105299764,
+      "grad_norm": 0.197265625,
+      "learning_rate": 1.3757128947002366e-06,
+      "loss": 1.058,
+      "step": 6200
+    },
+    {
+      "epoch": 0.865906245653081,
+      "grad_norm": 0.322265625,
+      "learning_rate": 1.340937543469189e-06,
+      "loss": 0.9862,
+      "step": 6225
+    },
+    {
+      "epoch": 0.8693837807761858,
+      "grad_norm": 0.322265625,
+      "learning_rate": 1.3061621922381418e-06,
+      "loss": 0.9465,
+      "step": 6250
+    },
+    {
+      "epoch": 0.8728613158992906,
+      "grad_norm": 0.1123046875,
+      "learning_rate": 1.2713868410070944e-06,
+      "loss": 0.9073,
+      "step": 6275
+    },
+    {
+      "epoch": 0.8763388510223953,
+      "grad_norm": 0.1943359375,
+      "learning_rate": 1.236611489776047e-06,
+      "loss": 0.8929,
+      "step": 6300
+    },
+    {
+      "epoch": 0.8798163861455001,
+      "grad_norm": 0.421875,
+      "learning_rate": 1.2018361385449993e-06,
+      "loss": 0.9447,
+      "step": 6325
+    },
+    {
+      "epoch": 0.8832939212686048,
+      "grad_norm": 0.1259765625,
+      "learning_rate": 1.1670607873139519e-06,
+      "loss": 0.9536,
+      "step": 6350
+    },
+    {
+      "epoch": 0.8867714563917095,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 1.1322854360829045e-06,
+      "loss": 0.9508,
+      "step": 6375
+    },
+    {
+      "epoch": 0.8902489915148143,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 1.097510084851857e-06,
+      "loss": 0.9177,
+      "step": 6400
+    },
+    {
+      "epoch": 0.8937265266379191,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 1.0627347336208096e-06,
+      "loss": 0.9255,
+      "step": 6425
+    },
+    {
+      "epoch": 0.8972040617610237,
+      "grad_norm": 0.267578125,
+      "learning_rate": 1.0279593823897622e-06,
+      "loss": 0.9855,
+      "step": 6450
+    },
+    {
+      "epoch": 0.9006815968841285,
+      "grad_norm": 2.03125,
+      "learning_rate": 9.931840311587148e-07,
+      "loss": 0.9591,
+      "step": 6475
+    },
+    {
+      "epoch": 0.9041591320072333,
+      "grad_norm": 0.56640625,
+      "learning_rate": 9.584086799276674e-07,
+      "loss": 0.929,
+      "step": 6500
+    },
+    {
+      "epoch": 0.907636667130338,
+      "grad_norm": 0.2236328125,
+      "learning_rate": 9.236333286966199e-07,
+      "loss": 0.9179,
+      "step": 6525
+    },
+    {
+      "epoch": 0.9111142022534428,
+      "grad_norm": 0.6171875,
+      "learning_rate": 8.888579774655724e-07,
+      "loss": 0.9565,
+      "step": 6550
+    },
+    {
+      "epoch": 0.9145917373765475,
+      "grad_norm": 0.2021484375,
+      "learning_rate": 8.540826262345251e-07,
+      "loss": 0.8986,
+      "step": 6575
+    },
+    {
+      "epoch": 0.9180692724996522,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 8.193072750034777e-07,
+      "loss": 0.9654,
+      "step": 6600
+    },
+    {
+      "epoch": 0.921546807622757,
+      "grad_norm": 0.23046875,
+      "learning_rate": 7.845319237724301e-07,
+      "loss": 1.0365,
+      "step": 6625
+    },
+    {
+      "epoch": 0.9250243427458618,
+      "grad_norm": 0.1875,
+      "learning_rate": 7.497565725413827e-07,
+      "loss": 0.9217,
+      "step": 6650
+    },
+    {
+      "epoch": 0.9285018778689664,
+      "grad_norm": 0.2412109375,
+      "learning_rate": 7.149812213103353e-07,
+      "loss": 1.0513,
+      "step": 6675
+    },
+    {
+      "epoch": 0.9319794129920712,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 6.802058700792878e-07,
+      "loss": 0.9691,
+      "step": 6700
+    },
+    {
+      "epoch": 0.935456948115176,
+      "grad_norm": 0.1796875,
+      "learning_rate": 6.454305188482404e-07,
+      "loss": 0.9488,
+      "step": 6725
+    },
+    {
+      "epoch": 0.9389344832382807,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 6.106551676171929e-07,
+      "loss": 0.9356,
+      "step": 6750
+    },
+    {
+      "epoch": 0.9424120183613854,
+      "grad_norm": 0.236328125,
+      "learning_rate": 5.758798163861455e-07,
+      "loss": 0.9483,
+      "step": 6775
+    },
+    {
+      "epoch": 0.9458895534844902,
+      "grad_norm": 0.11474609375,
+      "learning_rate": 5.411044651550981e-07,
+      "loss": 0.8652,
+      "step": 6800
+    },
+    {
+      "epoch": 0.9493670886075949,
+      "grad_norm": 0.23828125,
+      "learning_rate": 5.063291139240507e-07,
+      "loss": 0.9684,
+      "step": 6825
+    },
+    {
+      "epoch": 0.9528446237306997,
+      "grad_norm": 0.29296875,
+      "learning_rate": 4.7155376269300323e-07,
+      "loss": 0.961,
+      "step": 6850
+    },
+    {
+      "epoch": 0.9563221588538044,
+      "grad_norm": 0.306640625,
+      "learning_rate": 4.367784114619558e-07,
+      "loss": 0.9501,
+      "step": 6875
+    },
+    {
+      "epoch": 0.9597996939769091,
+      "grad_norm": 0.150390625,
+      "learning_rate": 4.0200306023090834e-07,
+      "loss": 0.9993,
+      "step": 6900
+    },
+    {
+      "epoch": 0.9632772291000139,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 3.672277089998609e-07,
+      "loss": 0.8676,
+      "step": 6925
+    },
+    {
+      "epoch": 0.9667547642231187,
+      "grad_norm": 0.185546875,
+      "learning_rate": 3.324523577688135e-07,
+      "loss": 1.0496,
+      "step": 6950
+    },
+    {
+      "epoch": 0.9702322993462233,
+      "grad_norm": 0.142578125,
+      "learning_rate": 2.9767700653776607e-07,
+      "loss": 0.9382,
+      "step": 6975
+    },
+    {
+      "epoch": 0.9737098344693281,
+      "grad_norm": 0.1416015625,
+      "learning_rate": 2.6290165530671865e-07,
+      "loss": 0.9294,
+      "step": 7000
+    },
+    {
+      "epoch": 0.9771873695924329,
+      "grad_norm": 0.1181640625,
+      "learning_rate": 2.2812630407567118e-07,
+      "loss": 1.0387,
+      "step": 7025
+    },
+    {
+      "epoch": 0.9806649047155376,
+      "grad_norm": 0.26171875,
+      "learning_rate": 1.9335095284462375e-07,
+      "loss": 0.9241,
+      "step": 7050
+    },
+    {
+      "epoch": 0.9841424398386424,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 1.585756016135763e-07,
+      "loss": 0.9952,
+      "step": 7075
+    },
+    {
+      "epoch": 0.9876199749617471,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 1.2380025038252888e-07,
+      "loss": 0.9206,
+      "step": 7100
+    },
+    {
+      "epoch": 0.9910975100848518,
+      "grad_norm": 0.1640625,
+      "learning_rate": 8.902489915148144e-08,
+      "loss": 0.9654,
+      "step": 7125
+    },
+    {
+      "epoch": 0.9945750452079566,
+      "grad_norm": 0.2197265625,
+      "learning_rate": 5.4249547920434e-08,
+      "loss": 0.9303,
+      "step": 7150
+    },
+    {
+      "epoch": 0.9980525803310614,
+      "grad_norm": 0.1220703125,
+      "learning_rate": 1.9474196689386566e-08,
+      "loss": 0.9058,
+      "step": 7175
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 7189,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.082170390057009e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

original_model/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5446fd9998148f8ee3d39578c23450b5e1fa84122a30263614b2baf55dc470cb
+size 5496