Upload 15 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,12 @@
+{%- set sys = 
+   "You are an expert in advanced master-level STEM multiple-choice questions.\n"
+   ~ "When given a question with choices A through D:\n"
+   ~ "1. Do NOT output any chain-of-thought or explanation.\n"
+   ~ "2. ONLY output exactly one of these four tokens (including the leading space): \" A\", \" B\", \" C\", or \" D\".\n"
+   ~ "3. Do not add any punctuation or extra text before or after."
+-%}
+{%- set usr = messages[0].content -%}
+
+{{- "<|im_start|>system\n" ~ sys ~ "\n<|im_end|>\n" -}}
+{{- "<|im_start|>user\n"  ~ usr ~ "\n<|im_end|>\n" -}}
+{{- "<|im_start|>assistant\nAnswer:" -}}

config.json

@@ -0,0 +1,30 @@
+{
+  "architectures": [
+    "Qwen3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "max_position_embeddings": 40960,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.52.4",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "temperature": 0.6,
+  "top_k": 20,
+  "top_p": 0.95,
+  "transformers_version": "4.52.4"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84b8430cf3ec4d1ae187ea4641879436a61daba568740b5adde9d31402e13ab9
+size 2384234968

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63abf194882a111351053589dd04a9126072d17008693e7b815dc2be25131eee
+size 2384371002

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b11a10749bfb1630d95eef94125f4590e8610c579d217d1f158e71ce518d72b
+size 14244

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2f4439f4ba5508cd783a6375abeef83d1c141c3ad8f898dc81b5f15b50a3575
+size 1064

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

tokenizer_config.json

@@ -0,0 +1,246 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "max_length": 2048,
+  "model_max_length": 131072,
+  "pad_to_multiple_of": null,
+  "pad_token": "<|endoftext|>",
+  "pad_token_type_id": 0,
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "stride": 0,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "truncation_side": "left",
+  "truncation_strategy": "longest_first",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,733 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.7248506220001959,
+  "eval_steps": 200,
+  "global_step": 1850,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.019590557351356647,
+      "grad_norm": 152.70944213867188,
+      "learning_rate": 3.2666666666666663e-07,
+      "logits/chosen": -2.876490592956543,
+      "logits/rejected": -2.730497121810913,
+      "logps/chosen": -467.5262756347656,
+      "logps/rejected": -378.40655517578125,
+      "loss": 0.64,
+      "rewards/accuracies": 0.6175000071525574,
+      "rewards/chosen": 0.30685558915138245,
+      "rewards/margins": 0.14467713236808777,
+      "rewards/rejected": 0.16217847168445587,
+      "step": 50
+    },
+    {
+      "epoch": 0.03918111470271329,
+      "grad_norm": 169.46041870117188,
+      "learning_rate": 6.6e-07,
+      "logits/chosen": -2.9327621459960938,
+      "logits/rejected": -2.691812038421631,
+      "logps/chosen": -459.0450439453125,
+      "logps/rejected": -358.7035217285156,
+      "loss": 0.5732,
+      "rewards/accuracies": 0.7200000286102295,
+      "rewards/chosen": 0.8860471844673157,
+      "rewards/margins": 0.42572346329689026,
+      "rewards/rejected": 0.4603237509727478,
+      "step": 100
+    },
+    {
+      "epoch": 0.05877167205406994,
+      "grad_norm": 163.8626708984375,
+      "learning_rate": 9.933333333333333e-07,
+      "logits/chosen": -2.9648749828338623,
+      "logits/rejected": -2.746995449066162,
+      "logps/chosen": -450.51348876953125,
+      "logps/rejected": -383.9017333984375,
+      "loss": 0.5708,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 1.1387741565704346,
+      "rewards/margins": 0.5126526355743408,
+      "rewards/rejected": 0.626121461391449,
+      "step": 150
+    },
+    {
+      "epoch": 0.07836222940542659,
+      "grad_norm": 101.9367446899414,
+      "learning_rate": 9.99894936347371e-07,
+      "logits/chosen": -3.0323171615600586,
+      "logits/rejected": -2.8769636154174805,
+      "logps/chosen": -467.7211608886719,
+      "logps/rejected": -375.92816162109375,
+      "loss": 0.5266,
+      "rewards/accuracies": 0.7174999713897705,
+      "rewards/chosen": 1.2015265226364136,
+      "rewards/margins": 0.7129627466201782,
+      "rewards/rejected": 0.48856377601623535,
+      "step": 200
+    },
+    {
+      "epoch": 0.07836222940542659,
+      "eval_logits/chosen": -2.2541391849517822,
+      "eval_logits/rejected": -2.285670042037964,
+      "eval_logps/chosen": -172.0572509765625,
+      "eval_logps/rejected": -200.56900024414062,
+      "eval_loss": 0.6408354640007019,
+      "eval_rewards/accuracies": 0.7420290112495422,
+      "eval_rewards/chosen": 0.0233369879424572,
+      "eval_rewards/margins": 0.1684449017047882,
+      "eval_rewards/rejected": -0.1451079100370407,
+      "eval_runtime": 45.4466,
+      "eval_samples_per_second": 15.183,
+      "eval_steps_per_second": 7.591,
+      "step": 200
+    },
+    {
+      "epoch": 0.09795278675678323,
+      "grad_norm": 106.43968200683594,
+      "learning_rate": 9.995711712979657e-07,
+      "logits/chosen": -2.953029155731201,
+      "logits/rejected": -2.7987329959869385,
+      "logps/chosen": -462.841796875,
+      "logps/rejected": -378.1797790527344,
+      "loss": 0.4989,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 1.3110496997833252,
+      "rewards/margins": 0.854219377040863,
+      "rewards/rejected": 0.4568302035331726,
+      "step": 250
+    },
+    {
+      "epoch": 0.11754334410813988,
+      "grad_norm": 105.10195922851562,
+      "learning_rate": 9.990288027658056e-07,
+      "logits/chosen": -2.9163262844085693,
+      "logits/rejected": -2.7455270290374756,
+      "logps/chosen": -438.2953186035156,
+      "logps/rejected": -367.7381591796875,
+      "loss": 0.5314,
+      "rewards/accuracies": 0.7400000095367432,
+      "rewards/chosen": 1.436720848083496,
+      "rewards/margins": 0.8745988607406616,
+      "rewards/rejected": 0.5621219277381897,
+      "step": 300
+    },
+    {
+      "epoch": 0.13713390145949653,
+      "grad_norm": 73.22066497802734,
+      "learning_rate": 9.982680680817391e-07,
+      "logits/chosen": -2.966728448867798,
+      "logits/rejected": -2.765995740890503,
+      "logps/chosen": -447.7353210449219,
+      "logps/rejected": -349.2236633300781,
+      "loss": 0.4596,
+      "rewards/accuracies": 0.7900000214576721,
+      "rewards/chosen": 1.517472267150879,
+      "rewards/margins": 1.1284364461898804,
+      "rewards/rejected": 0.3890356719493866,
+      "step": 350
+    },
+    {
+      "epoch": 0.15672445881085317,
+      "grad_norm": 193.4586181640625,
+      "learning_rate": 9.972893001297698e-07,
+      "logits/chosen": -2.92069935798645,
+      "logits/rejected": -2.7497365474700928,
+      "logps/chosen": -460.17755126953125,
+      "logps/rejected": -371.06011962890625,
+      "loss": 0.51,
+      "rewards/accuracies": 0.7350000143051147,
+      "rewards/chosen": 1.5156035423278809,
+      "rewards/margins": 1.070378065109253,
+      "rewards/rejected": 0.44522520899772644,
+      "step": 400
+    },
+    {
+      "epoch": 0.15672445881085317,
+      "eval_logits/chosen": -2.2648189067840576,
+      "eval_logits/rejected": -2.2913312911987305,
+      "eval_logps/chosen": -172.1621856689453,
+      "eval_logps/rejected": -201.6964874267578,
+      "eval_loss": 0.6150330305099487,
+      "eval_rewards/accuracies": 0.7710145115852356,
+      "eval_rewards/chosen": 0.012844223529100418,
+      "eval_rewards/margins": 0.2707003951072693,
+      "eval_rewards/rejected": -0.25785619020462036,
+      "eval_runtime": 45.871,
+      "eval_samples_per_second": 15.042,
+      "eval_steps_per_second": 7.521,
+      "step": 400
+    },
+    {
+      "epoch": 0.17631501616220982,
+      "grad_norm": 142.24362182617188,
+      "learning_rate": 9.96092927201391e-07,
+      "logits/chosen": -2.922132968902588,
+      "logits/rejected": -2.754628896713257,
+      "logps/chosen": -446.7215270996094,
+      "logps/rejected": -385.2563171386719,
+      "loss": 0.4712,
+      "rewards/accuracies": 0.7574999928474426,
+      "rewards/chosen": 1.4054139852523804,
+      "rewards/margins": 1.1510262489318848,
+      "rewards/rejected": 0.25438761711120605,
+      "step": 450
+    },
+    {
+      "epoch": 0.19590557351356647,
+      "grad_norm": 99.2269515991211,
+      "learning_rate": 9.946794728081737e-07,
+      "logits/chosen": -3.007890224456787,
+      "logits/rejected": -2.8413093090057373,
+      "logps/chosen": -465.13348388671875,
+      "logps/rejected": -390.49591064453125,
+      "loss": 0.4768,
+      "rewards/accuracies": 0.7799999713897705,
+      "rewards/chosen": 1.604026198387146,
+      "rewards/margins": 1.1249028444290161,
+      "rewards/rejected": 0.47912338376045227,
+      "step": 500
+    },
+    {
+      "epoch": 0.2154961308649231,
+      "grad_norm": 169.93678283691406,
+      "learning_rate": 9.930495554526878e-07,
+      "logits/chosen": -2.952453851699829,
+      "logits/rejected": -2.812680959701538,
+      "logps/chosen": -449.0242919921875,
+      "logps/rejected": -357.1375732421875,
+      "loss": 0.4962,
+      "rewards/accuracies": 0.7724999785423279,
+      "rewards/chosen": 1.6039185523986816,
+      "rewards/margins": 1.0116550922393799,
+      "rewards/rejected": 0.5922635793685913,
+      "step": 550
+    },
+    {
+      "epoch": 0.23508668821627976,
+      "grad_norm": 272.4690856933594,
+      "learning_rate": 9.912038883578552e-07,
+      "logits/chosen": -2.9756484031677246,
+      "logits/rejected": -2.760322332382202,
+      "logps/chosen": -460.68865966796875,
+      "logps/rejected": -377.5042724609375,
+      "loss": 0.5151,
+      "rewards/accuracies": 0.7799999713897705,
+      "rewards/chosen": 1.6494457721710205,
+      "rewards/margins": 1.0801762342453003,
+      "rewards/rejected": 0.5692696571350098,
+      "step": 600
+    },
+    {
+      "epoch": 0.23508668821627976,
+      "eval_logits/chosen": -2.328936815261841,
+      "eval_logits/rejected": -2.3488407135009766,
+      "eval_logps/chosen": -173.3373565673828,
+      "eval_logps/rejected": -202.97079467773438,
+      "eval_loss": 0.6263740062713623,
+      "eval_rewards/accuracies": 0.7507246136665344,
+      "eval_rewards/chosen": -0.10467354953289032,
+      "eval_rewards/margins": 0.2806117534637451,
+      "eval_rewards/rejected": -0.38528531789779663,
+      "eval_runtime": 45.7131,
+      "eval_samples_per_second": 15.094,
+      "eval_steps_per_second": 7.547,
+      "step": 600
+    },
+    {
+      "epoch": 0.2546772455676364,
+      "grad_norm": 72.5123291015625,
+      "learning_rate": 9.891432791548562e-07,
+      "logits/chosen": -2.9478814601898193,
+      "logits/rejected": -2.767559766769409,
+      "logps/chosen": -429.18499755859375,
+      "logps/rejected": -369.6413269042969,
+      "loss": 0.4703,
+      "rewards/accuracies": 0.7649999856948853,
+      "rewards/chosen": 1.570642113685608,
+      "rewards/margins": 1.231786847114563,
+      "rewards/rejected": 0.3388552963733673,
+      "step": 650
+    },
+    {
+      "epoch": 0.27426780291899305,
+      "grad_norm": 146.4647979736328,
+      "learning_rate": 9.868686295297258e-07,
+      "logits/chosen": -2.9772324562072754,
+      "logits/rejected": -2.7840633392333984,
+      "logps/chosen": -452.4783630371094,
+      "logps/rejected": -379.3604736328125,
+      "loss": 0.4646,
+      "rewards/accuracies": 0.7674999833106995,
+      "rewards/chosen": 1.685501217842102,
+      "rewards/margins": 1.2647416591644287,
+      "rewards/rejected": 0.4207596480846405,
+      "step": 700
+    },
+    {
+      "epoch": 0.29385836027034967,
+      "grad_norm": 115.44526672363281,
+      "learning_rate": 9.8438093482879e-07,
+      "logits/chosen": -3.0253145694732666,
+      "logits/rejected": -2.8723714351654053,
+      "logps/chosen": -461.543212890625,
+      "logps/rejected": -378.514892578125,
+      "loss": 0.5084,
+      "rewards/accuracies": 0.7549999952316284,
+      "rewards/chosen": 1.5600999593734741,
+      "rewards/margins": 1.2082923650741577,
+      "rewards/rejected": 0.3518076241016388,
+      "step": 750
+    },
+    {
+      "epoch": 0.31344891762170635,
+      "grad_norm": 49.29530334472656,
+      "learning_rate": 9.81681283623121e-07,
+      "logits/chosen": -3.0215282440185547,
+      "logits/rejected": -2.831230401992798,
+      "logps/chosen": -469.9809875488281,
+      "logps/rejected": -369.4506530761719,
+      "loss": 0.4662,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 1.9505739212036133,
+      "rewards/margins": 1.3873133659362793,
+      "rewards/rejected": 0.5632606744766235,
+      "step": 800
+    },
+    {
+      "epoch": 0.31344891762170635,
+      "eval_logits/chosen": -2.373304843902588,
+      "eval_logits/rejected": -2.391378402709961,
+      "eval_logps/chosen": -174.7285614013672,
+      "eval_logps/rejected": -204.59425354003906,
+      "eval_loss": 0.6305972933769226,
+      "eval_rewards/accuracies": 0.7681159377098083,
+      "eval_rewards/chosen": -0.2437940239906311,
+      "eval_rewards/margins": 0.30383673310279846,
+      "eval_rewards/rejected": -0.547630786895752,
+      "eval_runtime": 45.7764,
+      "eval_samples_per_second": 15.073,
+      "eval_steps_per_second": 7.537,
+      "step": 800
+    },
+    {
+      "epoch": 0.33303947497306297,
+      "grad_norm": 144.90708923339844,
+      "learning_rate": 9.787708572321983e-07,
+      "logits/chosen": -2.910259485244751,
+      "logits/rejected": -2.762113332748413,
+      "logps/chosen": -452.485107421875,
+      "logps/rejected": -337.2020568847656,
+      "loss": 0.5209,
+      "rewards/accuracies": 0.7400000095367432,
+      "rewards/chosen": 2.068610906600952,
+      "rewards/margins": 1.3179844617843628,
+      "rewards/rejected": 0.7506266236305237,
+      "step": 850
+    },
+    {
+      "epoch": 0.35263003232441964,
+      "grad_norm": 130.08224487304688,
+      "learning_rate": 9.756509292069825e-07,
+      "logits/chosen": -2.9319863319396973,
+      "logits/rejected": -2.783926010131836,
+      "logps/chosen": -419.07110595703125,
+      "logps/rejected": -382.4843444824219,
+      "loss": 0.5003,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 1.7963037490844727,
+      "rewards/margins": 1.2497668266296387,
+      "rewards/rejected": 0.5465368032455444,
+      "step": 900
+    },
+    {
+      "epoch": 0.37222058967577626,
+      "grad_norm": 151.3379669189453,
+      "learning_rate": 9.72322864772634e-07,
+      "logits/chosen": -3.0067360401153564,
+      "logits/rejected": -2.8380017280578613,
+      "logps/chosen": -463.267822265625,
+      "logps/rejected": -379.6971130371094,
+      "loss": 0.4351,
+      "rewards/accuracies": 0.8050000071525574,
+      "rewards/chosen": 1.8822674751281738,
+      "rewards/margins": 1.517218828201294,
+      "rewards/rejected": 0.36504870653152466,
+      "step": 950
+    },
+    {
+      "epoch": 0.39181114702713293,
+      "grad_norm": 127.39634704589844,
+      "learning_rate": 9.687881202311132e-07,
+      "logits/chosen": -2.975214958190918,
+      "logits/rejected": -2.7904016971588135,
+      "logps/chosen": -459.6684875488281,
+      "logps/rejected": -384.0487060546875,
+      "loss": 0.5026,
+      "rewards/accuracies": 0.7574999928474426,
+      "rewards/chosen": 2.054124355316162,
+      "rewards/margins": 1.4798840284347534,
+      "rewards/rejected": 0.5742404460906982,
+      "step": 1000
+    },
+    {
+      "epoch": 0.39181114702713293,
+      "eval_logits/chosen": -2.285802125930786,
+      "eval_logits/rejected": -2.3008267879486084,
+      "eval_logps/chosen": -173.751953125,
+      "eval_logps/rejected": -203.65634155273438,
+      "eval_loss": 0.6286672353744507,
+      "eval_rewards/accuracies": 0.7579709887504578,
+      "eval_rewards/chosen": -0.14613337814807892,
+      "eval_rewards/margins": 0.3077085614204407,
+      "eval_rewards/rejected": -0.453841894865036,
+      "eval_runtime": 45.6208,
+      "eval_samples_per_second": 15.125,
+      "eval_steps_per_second": 7.562,
+      "step": 1000
+    },
+    {
+      "epoch": 0.41140170437848955,
+      "grad_norm": 113.51675415039062,
+      "learning_rate": 9.65048242323929e-07,
+      "logits/chosen": -2.9107511043548584,
+      "logits/rejected": -2.766993522644043,
+      "logps/chosen": -419.68035888671875,
+      "logps/rejected": -354.5930480957031,
+      "loss": 0.4776,
+      "rewards/accuracies": 0.7825000286102295,
+      "rewards/chosen": 1.6637612581253052,
+      "rewards/margins": 1.2595511674880981,
+      "rewards/rejected": 0.40421026945114136,
+      "step": 1050
+    },
+    {
+      "epoch": 0.4309922617298462,
+      "grad_norm": 142.44851684570312,
+      "learning_rate": 9.611048675553127e-07,
+      "logits/chosen": -2.945481061935425,
+      "logits/rejected": -2.7998950481414795,
+      "logps/chosen": -473.3480529785156,
+      "logps/rejected": -378.8951416015625,
+      "loss": 0.4899,
+      "rewards/accuracies": 0.7799999713897705,
+      "rewards/chosen": 1.8652592897415161,
+      "rewards/margins": 1.3835867643356323,
+      "rewards/rejected": 0.48167240619659424,
+      "step": 1100
+    },
+    {
+      "epoch": 0.45058281908120285,
+      "grad_norm": 110.11934661865234,
+      "learning_rate": 9.569597214761124e-07,
+      "logits/chosen": -2.9105591773986816,
+      "logits/rejected": -2.7662746906280518,
+      "logps/chosen": -445.8592834472656,
+      "logps/rejected": -364.0835266113281,
+      "loss": 0.4642,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 1.9219467639923096,
+      "rewards/margins": 1.473950982093811,
+      "rewards/rejected": 0.4479956328868866,
+      "step": 1150
+    },
+    {
+      "epoch": 0.4701733764325595,
+      "grad_norm": 180.00526428222656,
+      "learning_rate": 9.526146179287222e-07,
+      "logits/chosen": -2.8630211353302,
+      "logits/rejected": -2.6804027557373047,
+      "logps/chosen": -449.76812744140625,
+      "logps/rejected": -364.3941650390625,
+      "loss": 0.5097,
+      "rewards/accuracies": 0.7325000166893005,
+      "rewards/chosen": 1.7920465469360352,
+      "rewards/margins": 1.2713873386383057,
+      "rewards/rejected": 0.5206592082977295,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4701733764325595,
+      "eval_logits/chosen": -2.2779555320739746,
+      "eval_logits/rejected": -2.2900030612945557,
+      "eval_logps/chosen": -173.5331573486328,
+      "eval_logps/rejected": -203.9847869873047,
+      "eval_loss": 0.6131945848464966,
+      "eval_rewards/accuracies": 0.7449275255203247,
+      "eval_rewards/chosen": -0.12425270676612854,
+      "eval_rewards/margins": 0.36243319511413574,
+      "eval_rewards/rejected": -0.4866859018802643,
+      "eval_runtime": 45.8929,
+      "eval_samples_per_second": 15.035,
+      "eval_steps_per_second": 7.517,
+      "step": 1200
+    },
+    {
+      "epoch": 0.48976393378391614,
+      "grad_norm": 160.4936065673828,
+      "learning_rate": 9.480714582533773e-07,
+      "logits/chosen": -2.8644652366638184,
+      "logits/rejected": -2.7200751304626465,
+      "logps/chosen": -454.3363952636719,
+      "logps/rejected": -376.8122253417969,
+      "loss": 0.4958,
+      "rewards/accuracies": 0.7524999976158142,
+      "rewards/chosen": 2.13558292388916,
+      "rewards/margins": 1.546592354774475,
+      "rewards/rejected": 0.5889905691146851,
+      "step": 1250
+    },
+    {
+      "epoch": 0.5093544911352728,
+      "grad_norm": 108.44024658203125,
+      "learning_rate": 9.433322304561614e-07,
+      "logits/chosen": -2.942742347717285,
+      "logits/rejected": -2.7668235301971436,
+      "logps/chosen": -450.0126647949219,
+      "logps/rejected": -390.0963134765625,
+      "loss": 0.4247,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 1.9560456275939941,
+      "rewards/margins": 1.6855427026748657,
+      "rewards/rejected": 0.27050289511680603,
+      "step": 1300
+    },
+    {
+      "epoch": 0.5289450484866295,
+      "grad_norm": 280.08380126953125,
+      "learning_rate": 9.383990083390903e-07,
+      "logits/chosen": -2.957747220993042,
+      "logits/rejected": -2.795564889907837,
+      "logps/chosen": -453.2030334472656,
+      "logps/rejected": -413.6679992675781,
+      "loss": 0.4633,
+      "rewards/accuracies": 0.7674999833106995,
+      "rewards/chosen": 1.6891446113586426,
+      "rewards/margins": 1.4608780145645142,
+      "rewards/rejected": 0.22826646268367767,
+      "step": 1350
+    },
+    {
+      "epoch": 0.5485356058379861,
+      "grad_norm": 214.8365478515625,
+      "learning_rate": 9.332739505926528e-07,
+      "logits/chosen": -2.917468309402466,
+      "logits/rejected": -2.753814697265625,
+      "logps/chosen": -471.89483642578125,
+      "logps/rejected": -402.4897155761719,
+      "loss": 0.4122,
+      "rewards/accuracies": 0.8025000095367432,
+      "rewards/chosen": 1.7866332530975342,
+      "rewards/margins": 1.777306318283081,
+      "rewards/rejected": 0.009327063336968422,
+      "step": 1400
+    },
+    {
+      "epoch": 0.5485356058379861,
+      "eval_logits/chosen": -2.384613275527954,
+      "eval_logits/rejected": -2.4019815921783447,
+      "eval_logps/chosen": -176.3463592529297,
+      "eval_logps/rejected": -207.75599670410156,
+      "eval_loss": 0.5938982367515564,
+      "eval_rewards/accuracies": 0.7782608866691589,
+      "eval_rewards/chosen": -0.40557393431663513,
+      "eval_rewards/margins": 0.4582330286502838,
+      "eval_rewards/rejected": -0.863806962966919,
+      "eval_runtime": 45.9899,
+      "eval_samples_per_second": 15.003,
+      "eval_steps_per_second": 7.502,
+      "step": 1400
+    },
+    {
+      "epoch": 0.5681261631893427,
+      "grad_norm": 143.27638244628906,
+      "learning_rate": 9.279592998512064e-07,
+      "logits/chosen": -2.954049587249756,
+      "logits/rejected": -2.828245162963867,
+      "logps/chosen": -467.3232116699219,
+      "logps/rejected": -384.35601806640625,
+      "loss": 0.4616,
+      "rewards/accuracies": 0.7900000214576721,
+      "rewards/chosen": 1.6394932270050049,
+      "rewards/margins": 1.5439642667770386,
+      "rewards/rejected": 0.09552911669015884,
+      "step": 1450
+    },
+    {
+      "epoch": 0.5877167205406993,
+      "grad_norm": 70.37281799316406,
+      "learning_rate": 9.224573817116396e-07,
+      "logits/chosen": -2.9961767196655273,
+      "logits/rejected": -2.8868143558502197,
+      "logps/chosen": -478.9124755859375,
+      "logps/rejected": -402.163330078125,
+      "loss": 0.3946,
+      "rewards/accuracies": 0.8174999952316284,
+      "rewards/chosen": 2.2281653881073,
+      "rewards/margins": 2.0543277263641357,
+      "rewards/rejected": 0.17383766174316406,
+      "step": 1500
+    },
+    {
+      "epoch": 0.6073072778920561,
+      "grad_norm": 116.5025863647461,
+      "learning_rate": 9.16770603715733e-07,
+      "logits/chosen": -2.935271978378296,
+      "logits/rejected": -2.8107409477233887,
+      "logps/chosen": -448.86444091796875,
+      "logps/rejected": -377.48944091796875,
+      "loss": 0.5143,
+      "rewards/accuracies": 0.7599999904632568,
+      "rewards/chosen": 1.8645941019058228,
+      "rewards/margins": 1.5373382568359375,
+      "rewards/rejected": 0.3272559642791748,
+      "step": 1550
+    },
+    {
+      "epoch": 0.6268978352434127,
+      "grad_norm": 115.91889190673828,
+      "learning_rate": 9.109014542966609e-07,
+      "logits/chosen": -3.030916452407837,
+      "logits/rejected": -2.8699285984039307,
+      "logps/chosen": -430.56219482421875,
+      "logps/rejected": -374.2171936035156,
+      "loss": 0.5026,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 1.8679478168487549,
+      "rewards/margins": 1.4932770729064941,
+      "rewards/rejected": 0.3746708631515503,
+      "step": 1600
+    },
+    {
+      "epoch": 0.6268978352434127,
+      "eval_logits/chosen": -2.468580484390259,
+      "eval_logits/rejected": -2.4822895526885986,
+      "eval_logps/chosen": -177.13941955566406,
+      "eval_logps/rejected": -209.1125030517578,
+      "eval_loss": 0.590223491191864,
+      "eval_rewards/accuracies": 0.7927536368370056,
+      "eval_rewards/chosen": -0.484881192445755,
+      "eval_rewards/margins": 0.514574408531189,
+      "eval_rewards/rejected": -0.9994555711746216,
+      "eval_runtime": 45.9912,
+      "eval_samples_per_second": 15.003,
+      "eval_steps_per_second": 7.501,
+      "step": 1600
+    },
+    {
+      "epoch": 0.6464883925947693,
+      "grad_norm": 144.60536193847656,
+      "learning_rate": 9.04852501690097e-07,
+      "logits/chosen": -3.0554418563842773,
+      "logits/rejected": -2.916806221008301,
+      "logps/chosen": -448.33453369140625,
+      "logps/rejected": -382.4766540527344,
+      "loss": 0.4249,
+      "rewards/accuracies": 0.7950000166893005,
+      "rewards/chosen": 1.6690967082977295,
+      "rewards/margins": 1.7366982698440552,
+      "rewards/rejected": -0.06760149449110031,
+      "step": 1650
+    },
+    {
+      "epoch": 0.6660789499461259,
+      "grad_norm": 204.64212036132812,
+      "learning_rate": 8.986263928104007e-07,
+      "logits/chosen": -2.9387013912200928,
+      "logits/rejected": -2.8714663982391357,
+      "logps/chosen": -462.238525390625,
+      "logps/rejected": -362.3359069824219,
+      "loss": 0.4699,
+      "rewards/accuracies": 0.7724999785423279,
+      "rewards/chosen": 2.0341713428497314,
+      "rewards/margins": 1.6428416967391968,
+      "rewards/rejected": 0.39132973551750183,
+      "step": 1700
+    },
+    {
+      "epoch": 0.6856695072974827,
+      "grad_norm": 158.81417846679688,
+      "learning_rate": 8.922258520923739e-07,
+      "logits/chosen": -2.928734064102173,
+      "logits/rejected": -2.7294297218322754,
+      "logps/chosen": -432.1619873046875,
+      "logps/rejected": -378.280517578125,
+      "loss": 0.5039,
+      "rewards/accuracies": 0.7699999809265137,
+      "rewards/chosen": 1.977944016456604,
+      "rewards/margins": 1.4837253093719482,
+      "rewards/rejected": 0.4942188262939453,
+      "step": 1750
+    },
+    {
+      "epoch": 0.7052600646488393,
+      "grad_norm": 128.84994506835938,
+      "learning_rate": 8.856536802990969e-07,
+      "logits/chosen": -2.9586968421936035,
+      "logits/rejected": -2.868234634399414,
+      "logps/chosen": -432.3280334472656,
+      "logps/rejected": -376.2671203613281,
+      "loss": 0.461,
+      "rewards/accuracies": 0.7799999713897705,
+      "rewards/chosen": 1.9399315118789673,
+      "rewards/margins": 1.5768976211547852,
+      "rewards/rejected": 0.36303389072418213,
+      "step": 1800
+    },
+    {
+      "epoch": 0.7052600646488393,
+      "eval_logits/chosen": -2.4123268127441406,
+      "eval_logits/rejected": -2.4266409873962402,
+      "eval_logps/chosen": -174.25506591796875,
+      "eval_logps/rejected": -205.96014404296875,
+      "eval_loss": 0.5846713185310364,
+      "eval_rewards/accuracies": 0.8202898502349854,
+      "eval_rewards/chosen": -0.19644570350646973,
+      "eval_rewards/margins": 0.4877746105194092,
+      "eval_rewards/rejected": -0.6842203140258789,
+      "eval_runtime": 45.7164,
+      "eval_samples_per_second": 15.093,
+      "eval_steps_per_second": 7.547,
+      "step": 1800
+    },
+    {
+      "epoch": 0.7248506220001959,
+      "grad_norm": 227.02520751953125,
+      "learning_rate": 8.789127532963639e-07,
+      "logits/chosen": -2.9502925872802734,
+      "logits/rejected": -2.8222861289978027,
+      "logps/chosen": -459.9781188964844,
+      "logps/rejected": -375.967041015625,
+      "loss": 0.3961,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 2.2383108139038086,
+      "rewards/margins": 1.8207529783248901,
+      "rewards/rejected": 0.4175575375556946,
+      "step": 1850
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 7659,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 370,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a21ff3ab98891da496fbd96dbfe051af3d4396f06537d54a7d17190856993db
+size 6200