Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1_epoch3/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/raw_datasets.json filter=lfs diff=lfs merge=lfs -text
+Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/tokenizer.json filter=lfs diff=lfs merge=lfs -text

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1/README.md

@@ -0,0 +1,59 @@
+---
+library_name: transformers
+base_model: /home/v-liuzichuan/huggingface/Llama-3.2-3B-Instruct
+tags:
+- alignment-handbook
+- generated_from_trainer
+datasets:
+- /home/v-liuzichuan/huggingface/princeton-nlp/llama3-math
+model-index:
+- name: Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1
+
+This model is a fine-tuned version of [/home/v-liuzichuan/huggingface/Llama-3.2-3B-Instruct](https://huggingface.co//home/v-liuzichuan/huggingface/Llama-3.2-3B-Instruct) on the /home/v-liuzichuan/huggingface/princeton-nlp/llama3-math dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- gradient_accumulation_steps: 64
+- total_train_batch_size: 128
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.45.2
+- Pytorch 2.5.1+cu121
+- Datasets 3.5.0
+- Tokenizers 0.20.3

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1/all_results.json

@@ -0,0 +1,24 @@
+{
+    "epoch": 0.9880609304240429,
+    "eval_logits/chosen": -0.006348939146846533,
+    "eval_logits/rejected": 0.15363818407058716,
+    "eval_logps/chosen": -152.83180236816406,
+    "eval_logps/ref_chosen": -143.584228515625,
+    "eval_logps/ref_rejected": -180.65379333496094,
+    "eval_logps/rejected": -193.19842529296875,
+    "eval_loss": 2200.785888671875,
+    "eval_rewards/accuracies": 0.7675926089286804,
+    "eval_rewards/chosen": -0.09247560054063797,
+    "eval_rewards/margins": 0.03297054395079613,
+    "eval_rewards/rejected": -0.125446155667305,
+    "eval_runtime": 144.1905,
+    "eval_samples": 1080,
+    "eval_samples_per_second": 7.49,
+    "eval_steps_per_second": 1.873,
+    "total_flos": 0.0,
+    "train_loss": 2318.3018196614585,
+    "train_runtime": 5362.5076,
+    "train_samples": 9715,
+    "train_samples_per_second": 1.812,
+    "train_steps_per_second": 0.014
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1/config.json

@@ -0,0 +1,40 @@
+{
+  "_name_or_path": "/home/v-liuzichuan/huggingface/Llama-3.2-3B-Instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.2",
+  "use_cache": true,
+  "vocab_size": 128256
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1/eval_results.json

@@ -0,0 +1,18 @@
+{
+    "epoch": 0.9880609304240429,
+    "eval_logits/chosen": -0.006348939146846533,
+    "eval_logits/rejected": 0.15363818407058716,
+    "eval_logps/chosen": -152.83180236816406,
+    "eval_logps/ref_chosen": -143.584228515625,
+    "eval_logps/ref_rejected": -180.65379333496094,
+    "eval_logps/rejected": -193.19842529296875,
+    "eval_loss": 2200.785888671875,
+    "eval_rewards/accuracies": 0.7675926089286804,
+    "eval_rewards/chosen": -0.09247560054063797,
+    "eval_rewards/margins": 0.03297054395079613,
+    "eval_rewards/rejected": -0.125446155667305,
+    "eval_runtime": 144.1905,
+    "eval_samples": 1080,
+    "eval_samples_per_second": 7.49,
+    "eval_steps_per_second": 1.873
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.45.2"
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:406950142942bd6ecc5e4d0189e5536f9c55b4d0327a287472bba05b6d39951e
+size 4965799096

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b36468d396c14af5406e75ba9946c39ed91227034dfee3da077dcf3eacdc9ae
+size 2247734992

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1/model.safetensors.index.json

@@ -0,0 +1,262 @@
+{
+  "metadata": {
+    "total_size": 7213504512
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|eot_id|>"
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- if strftime_now is defined %}\n        {%- set date_string = strftime_now(\"%d %b %Y\") %}\n    {%- else %}\n        {%- set date_string = \"26 Jul 2024\" %}\n    {%- endif %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n        {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n        {{- '\"parameters\": ' }}\n        {{- tool_call.arguments | tojson }}\n        {{- \"}\" }}\n        {{- \"<|eot_id|>\" }}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 2048,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9880609304240429,
+    "total_flos": 0.0,
+    "train_loss": 2318.3018196614585,
+    "train_runtime": 5362.5076,
+    "train_samples": 9715,
+    "train_samples_per_second": 1.812,
+    "train_steps_per_second": 0.014
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1/trainer_state.json

@@ -0,0 +1,1317 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9880609304240429,
+  "eval_steps": 500,
+  "global_step": 75,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.013174145738987238,
+      "grad_norm": 6860.466796875,
+      "learning_rate": 6.25e-08,
+      "logits/chosen": 0.141204833984375,
+      "logits/rejected": 0.33030927181243896,
+      "logps/chosen": -145.53533935546875,
+      "logps/ref_chosen": -145.53533935546875,
+      "logps/ref_rejected": -193.32150268554688,
+      "logps/rejected": -193.32150268554688,
+      "loss": 2500.0,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.026348291477974475,
+      "grad_norm": 6303.73779296875,
+      "learning_rate": 1.25e-07,
+      "logits/chosen": 0.13193297386169434,
+      "logits/rejected": 0.2929544448852539,
+      "logps/chosen": -140.2626953125,
+      "logps/ref_chosen": -140.2626953125,
+      "logps/ref_rejected": -176.88131713867188,
+      "logps/rejected": -176.88131713867188,
+      "loss": 2500.0,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 2
+    },
+    {
+      "epoch": 0.03952243721696171,
+      "grad_norm": 7013.66455078125,
+      "learning_rate": 1.875e-07,
+      "logits/chosen": 0.13777679204940796,
+      "logits/rejected": 0.3124394416809082,
+      "logps/chosen": -140.7003173828125,
+      "logps/ref_chosen": -140.56329345703125,
+      "logps/ref_rejected": -178.77528381347656,
+      "logps/rejected": -179.00807189941406,
+      "loss": 2490.6772,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.001370340003632009,
+      "rewards/margins": 0.0009573930292390287,
+      "rewards/rejected": -0.0023277332074940205,
+      "step": 3
+    },
+    {
+      "epoch": 0.05269658295594895,
+      "grad_norm": 6542.68701171875,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": 0.11986075341701508,
+      "logits/rejected": 0.3138517737388611,
+      "logps/chosen": -142.2206268310547,
+      "logps/ref_chosen": -142.1716766357422,
+      "logps/ref_rejected": -182.4105682373047,
+      "logps/rejected": -182.5343017578125,
+      "loss": 2492.9197,
+      "rewards/accuracies": 0.5859375,
+      "rewards/chosen": -0.0004894107696600258,
+      "rewards/margins": 0.0007477980107069016,
+      "rewards/rejected": -0.0012372087221592665,
+      "step": 4
+    },
+    {
+      "epoch": 0.06587072869493618,
+      "grad_norm": 6226.75830078125,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": 0.1316731721162796,
+      "logits/rejected": 0.27220460772514343,
+      "logps/chosen": -136.12843322753906,
+      "logps/ref_chosen": -136.04742431640625,
+      "logps/ref_rejected": -174.37258911132812,
+      "logps/rejected": -174.43295288085938,
+      "loss": 2502.3987,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": -0.0008100616978481412,
+      "rewards/margins": -0.00020654620311688632,
+      "rewards/rejected": -0.0006035154801793396,
+      "step": 5
+    },
+    {
+      "epoch": 0.07904487443392343,
+      "grad_norm": 6592.1669921875,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": 0.014960660599172115,
+      "logits/rejected": 0.2215287983417511,
+      "logps/chosen": -141.27008056640625,
+      "logps/ref_chosen": -141.18539428710938,
+      "logps/ref_rejected": -184.99798583984375,
+      "logps/rejected": -185.0793914794922,
+      "loss": 2500.5635,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": -0.0008468800224363804,
+      "rewards/margins": -3.290064341854304e-05,
+      "rewards/rejected": -0.0008139795390889049,
+      "step": 6
+    },
+    {
+      "epoch": 0.09221902017291066,
+      "grad_norm": 6696.421875,
+      "learning_rate": 4.375e-07,
+      "logits/chosen": 0.18217767775058746,
+      "logits/rejected": 0.29594191908836365,
+      "logps/chosen": -137.65072631835938,
+      "logps/ref_chosen": -137.64540100097656,
+      "logps/ref_rejected": -162.30291748046875,
+      "logps/rejected": -162.27566528320312,
+      "loss": 2503.5322,
+      "rewards/accuracies": 0.484375,
+      "rewards/chosen": -5.3269868658389896e-05,
+      "rewards/margins": -0.00032598141115158796,
+      "rewards/rejected": 0.00027271150611341,
+      "step": 7
+    },
+    {
+      "epoch": 0.1053931659118979,
+      "grad_norm": 6849.87255859375,
+      "learning_rate": 5e-07,
+      "logits/chosen": 0.14773835241794586,
+      "logits/rejected": 0.27819374203681946,
+      "logps/chosen": -142.74822998046875,
+      "logps/ref_chosen": -142.64285278320312,
+      "logps/ref_rejected": -179.400390625,
+      "logps/rejected": -179.61361694335938,
+      "loss": 2489.5378,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.0010537999914959073,
+      "rewards/margins": 0.0010784172918647528,
+      "rewards/rejected": -0.002132217399775982,
+      "step": 8
+    },
+    {
+      "epoch": 0.11856731165088513,
+      "grad_norm": 6715.9052734375,
+      "learning_rate": 4.997252228714278e-07,
+      "logits/chosen": 0.16841058433055878,
+      "logits/rejected": 0.25743114948272705,
+      "logps/chosen": -143.73524475097656,
+      "logps/ref_chosen": -143.59718322753906,
+      "logps/ref_rejected": -167.90943908691406,
+      "logps/rejected": -168.07098388671875,
+      "loss": 2497.9673,
+      "rewards/accuracies": 0.5703125,
+      "rewards/chosen": -0.001380585483275354,
+      "rewards/margins": 0.0002349048008909449,
+      "rewards/rejected": -0.0016154901823028922,
+      "step": 9
+    },
+    {
+      "epoch": 0.13174145738987236,
+      "grad_norm": 7364.2939453125,
+      "learning_rate": 4.989014955054745e-07,
+      "logits/chosen": 0.06786679476499557,
+      "logits/rejected": 0.271741658449173,
+      "logps/chosen": -136.30813598632812,
+      "logps/ref_chosen": -136.13136291503906,
+      "logps/ref_rejected": -167.5884246826172,
+      "logps/rejected": -167.917724609375,
+      "loss": 2485.0818,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0017676865682005882,
+      "rewards/margins": 0.001525329309515655,
+      "rewards/rejected": -0.003293015994131565,
+      "step": 10
+    },
+    {
+      "epoch": 0.14491560312885962,
+      "grad_norm": 6181.556640625,
+      "learning_rate": 4.975306286336627e-07,
+      "logits/chosen": 0.13534238934516907,
+      "logits/rejected": 0.2919628918170929,
+      "logps/chosen": -140.97483825683594,
+      "logps/ref_chosen": -140.63983154296875,
+      "logps/ref_rejected": -187.97640991210938,
+      "logps/rejected": -188.3951416015625,
+      "loss": 2491.9521,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": -0.0033500813879072666,
+      "rewards/margins": 0.0008371174335479736,
+      "rewards/rejected": -0.00418719882145524,
+      "step": 11
+    },
+    {
+      "epoch": 0.15808974886784685,
+      "grad_norm": 6728.86962890625,
+      "learning_rate": 4.956156357188939e-07,
+      "logits/chosen": 0.15034745633602142,
+      "logits/rejected": 0.2988182306289673,
+      "logps/chosen": -138.34246826171875,
+      "logps/ref_chosen": -137.91943359375,
+      "logps/ref_rejected": -173.40560913085938,
+      "logps/rejected": -174.0308837890625,
+      "loss": 2480.1863,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.004230341874063015,
+      "rewards/margins": 0.002022407716140151,
+      "rewards/rejected": -0.006252749357372522,
+      "step": 12
+    },
+    {
+      "epoch": 0.17126389460683408,
+      "grad_norm": 6973.720703125,
+      "learning_rate": 4.931607263312032e-07,
+      "logits/chosen": 0.20189328491687775,
+      "logits/rejected": 0.3522319495677948,
+      "logps/chosen": -140.02114868164062,
+      "logps/ref_chosen": -139.3595428466797,
+      "logps/ref_rejected": -178.2056121826172,
+      "logps/rejected": -179.11842346191406,
+      "loss": 2475.3787,
+      "rewards/accuracies": 0.6171875,
+      "rewards/chosen": -0.00661615002900362,
+      "rewards/margins": 0.0025120466016232967,
+      "rewards/rejected": -0.00912819616496563,
+      "step": 13
+    },
+    {
+      "epoch": 0.1844380403458213,
+      "grad_norm": 7222.79541015625,
+      "learning_rate": 4.9017129689421e-07,
+      "logits/chosen": 0.09214643388986588,
+      "logits/rejected": 0.28671640157699585,
+      "logps/chosen": -141.103271484375,
+      "logps/ref_chosen": -140.22817993164062,
+      "logps/ref_rejected": -174.04420471191406,
+      "logps/rejected": -174.9939727783203,
+      "loss": 2492.9717,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.008750887587666512,
+      "rewards/margins": 0.0007466625538654625,
+      "rewards/rejected": -0.009497551247477531,
+      "step": 14
+    },
+    {
+      "epoch": 0.19761218608480857,
+      "grad_norm": 9396.513671875,
+      "learning_rate": 4.866539188226085e-07,
+      "logits/chosen": 0.14158909022808075,
+      "logits/rejected": 0.22743283212184906,
+      "logps/chosen": -141.16696166992188,
+      "logps/ref_chosen": -140.21017456054688,
+      "logps/ref_rejected": -165.01068115234375,
+      "logps/rejected": -166.10667419433594,
+      "loss": 2486.5557,
+      "rewards/accuracies": 0.5859375,
+      "rewards/chosen": -0.009568018838763237,
+      "rewards/margins": 0.0013918390031903982,
+      "rewards/rejected": -0.010959859006106853,
+      "step": 15
+    },
+    {
+      "epoch": 0.2107863318237958,
+      "grad_norm": 6558.8779296875,
+      "learning_rate": 4.826163240767716e-07,
+      "logits/chosen": 0.1439928561449051,
+      "logits/rejected": 0.20027083158493042,
+      "logps/chosen": -135.67388916015625,
+      "logps/ref_chosen": -134.51571655273438,
+      "logps/ref_rejected": -160.91802978515625,
+      "logps/rejected": -162.40234375,
+      "loss": 2468.094,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.011581514962017536,
+      "rewards/margins": 0.0032616225071251392,
+      "rewards/rejected": -0.014843138866126537,
+      "step": 16
+    },
+    {
+      "epoch": 0.22396047756278303,
+      "grad_norm": 9624.171875,
+      "learning_rate": 4.780673881662242e-07,
+      "logits/chosen": 0.09058184921741486,
+      "logits/rejected": 0.30712011456489563,
+      "logps/chosen": -138.65167236328125,
+      "logps/ref_chosen": -137.29457092285156,
+      "logps/ref_rejected": -186.57139587402344,
+      "logps/rejected": -188.1879425048828,
+      "loss": 2475.0437,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.013570962473750114,
+      "rewards/margins": 0.0025946293026208878,
+      "rewards/rejected": -0.016165591776371002,
+      "step": 17
+    },
+    {
+      "epoch": 0.23713462330177026,
+      "grad_norm": 7499.15478515625,
+      "learning_rate": 4.730171106393466e-07,
+      "logits/chosen": 0.13107196986675262,
+      "logits/rejected": 0.23367160558700562,
+      "logps/chosen": -137.4244842529297,
+      "logps/ref_chosen": -135.82603454589844,
+      "logps/ref_rejected": -162.28997802734375,
+      "logps/rejected": -164.30157470703125,
+      "loss": 2459.6831,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.015984630212187767,
+      "rewards/margins": 0.004131443798542023,
+      "rewards/rejected": -0.02011607214808464,
+      "step": 18
+    },
+    {
+      "epoch": 0.2503087690407575,
+      "grad_norm": 6299.62109375,
+      "learning_rate": 4.6747659310219757e-07,
+      "logits/chosen": 0.17090073227882385,
+      "logits/rejected": 0.30368080735206604,
+      "logps/chosen": -144.9495849609375,
+      "logps/ref_chosen": -143.41900634765625,
+      "logps/ref_rejected": -173.9239501953125,
+      "logps/rejected": -176.06329345703125,
+      "loss": 2440.637,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.015305725857615471,
+      "rewards/margins": 0.006087546236813068,
+      "rewards/rejected": -0.021393269300460815,
+      "step": 19
+    },
+    {
+      "epoch": 0.2634829147797447,
+      "grad_norm": 7396.08349609375,
+      "learning_rate": 4.6145801481477433e-07,
+      "logits/chosen": 0.16462361812591553,
+      "logits/rejected": 0.30562156438827515,
+      "logps/chosen": -146.265625,
+      "logps/ref_chosen": -144.12081909179688,
+      "logps/ref_rejected": -185.55020141601562,
+      "logps/rejected": -188.32659912109375,
+      "loss": 2438.6282,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.02144799381494522,
+      "rewards/margins": 0.006316186860203743,
+      "rewards/rejected": -0.027764180675148964,
+      "step": 20
+    },
+    {
+      "epoch": 0.276657060518732,
+      "grad_norm": 5270.61865234375,
+      "learning_rate": 4.549746059183561e-07,
+      "logits/chosen": 0.15787144005298615,
+      "logits/rejected": 0.27715587615966797,
+      "logps/chosen": -139.85423278808594,
+      "logps/ref_chosen": -137.74465942382812,
+      "logps/ref_rejected": -166.4859619140625,
+      "logps/rejected": -169.4452667236328,
+      "loss": 2417.2112,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.0210958831012249,
+      "rewards/margins": 0.008497259579598904,
+      "rewards/rejected": -0.029593143612146378,
+      "step": 21
+    },
+    {
+      "epoch": 0.28983120625771924,
+      "grad_norm": 7718.4482421875,
+      "learning_rate": 4.480406183527823e-07,
+      "logits/chosen": 0.07220113277435303,
+      "logits/rejected": 0.29546162486076355,
+      "logps/chosen": -143.28038024902344,
+      "logps/ref_chosen": -140.8328094482422,
+      "logps/ref_rejected": -200.5518798828125,
+      "logps/rejected": -203.9213104248047,
+      "loss": 2410.8091,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.02447575330734253,
+      "rewards/margins": 0.009218500927090645,
+      "rewards/rejected": -0.033694252371788025,
+      "step": 22
+    },
+    {
+      "epoch": 0.3030053519967065,
+      "grad_norm": 6451.75341796875,
+      "learning_rate": 4.4067129452759546e-07,
+      "logits/chosen": 0.07104746997356415,
+      "logits/rejected": 0.2862705588340759,
+      "logps/chosen": -138.75106811523438,
+      "logps/ref_chosen": -136.00917053222656,
+      "logps/ref_rejected": -180.5914306640625,
+      "logps/rejected": -184.22340393066406,
+      "loss": 2414.3352,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.02741893380880356,
+      "rewards/margins": 0.008900967426598072,
+      "rewards/rejected": -0.036319900304079056,
+      "step": 23
+    },
+    {
+      "epoch": 0.3161794977356937,
+      "grad_norm": 5537.490234375,
+      "learning_rate": 4.3288283381591725e-07,
+      "logits/chosen": 0.12691445648670197,
+      "logits/rejected": 0.2488580346107483,
+      "logps/chosen": -146.79885864257812,
+      "logps/ref_chosen": -143.82127380371094,
+      "logps/ref_rejected": -182.13327026367188,
+      "logps/rejected": -186.1513671875,
+      "loss": 2399.553,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.02977580577135086,
+      "rewards/margins": 0.010404943488538265,
+      "rewards/rejected": -0.0401807501912117,
+      "step": 24
+    },
+    {
+      "epoch": 0.32935364347468093,
+      "grad_norm": 6902.609375,
+      "learning_rate": 4.246923569447104e-07,
+      "logits/chosen": 0.19381573796272278,
+      "logits/rejected": 0.31139570474624634,
+      "logps/chosen": -144.62013244628906,
+      "logps/ref_chosen": -141.2464141845703,
+      "logps/ref_rejected": -168.5917510986328,
+      "logps/rejected": -172.9250030517578,
+      "loss": 2407.001,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.03373728320002556,
+      "rewards/margins": 0.009595153853297234,
+      "rewards/rejected": -0.04333243519067764,
+      "step": 25
+    },
+    {
+      "epoch": 0.34252778921366817,
+      "grad_norm": 6792.0068359375,
+      "learning_rate": 4.161178683597054e-07,
+      "logits/chosen": 0.14972905814647675,
+      "logits/rejected": 0.267787367105484,
+      "logps/chosen": -148.2871551513672,
+      "logps/ref_chosen": -144.67782592773438,
+      "logps/ref_rejected": -183.55374145507812,
+      "logps/rejected": -188.48240661621094,
+      "loss": 2372.5386,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.03609343245625496,
+      "rewards/margins": 0.013193339109420776,
+      "rewards/rejected": -0.049286771565675735,
+      "step": 26
+    },
+    {
+      "epoch": 0.3557019349526554,
+      "grad_norm": 7997.36669921875,
+      "learning_rate": 4.0717821664772124e-07,
+      "logits/chosen": 0.11141426116228104,
+      "logits/rejected": 0.28967005014419556,
+      "logps/chosen": -142.55812072753906,
+      "logps/ref_chosen": -138.5041046142578,
+      "logps/ref_rejected": -177.208251953125,
+      "logps/rejected": -182.5889892578125,
+      "loss": 2372.2795,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.04054022952914238,
+      "rewards/margins": 0.013267194852232933,
+      "rewards/rejected": -0.053807422518730164,
+      "step": 27
+    },
+    {
+      "epoch": 0.3688760806916426,
+      "grad_norm": 7562.548828125,
+      "learning_rate": 3.978930531033806e-07,
+      "logits/chosen": 0.09050436317920685,
+      "logits/rejected": 0.2789982855319977,
+      "logps/chosen": -142.0299072265625,
+      "logps/ref_chosen": -137.58804321289062,
+      "logps/ref_rejected": -182.34164428710938,
+      "logps/rejected": -188.1849822998047,
+      "loss": 2365.6448,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": -0.04441874474287033,
+      "rewards/margins": 0.014014637097716331,
+      "rewards/rejected": -0.058433376252651215,
+      "step": 28
+    },
+    {
+      "epoch": 0.3820502264306299,
+      "grad_norm": 10295.1181640625,
+      "learning_rate": 3.882827885312998e-07,
+      "logits/chosen": 0.13718749582767487,
+      "logits/rejected": 0.2508697807788849,
+      "logps/chosen": -144.1764373779297,
+      "logps/ref_chosen": -139.355712890625,
+      "logps/ref_rejected": -166.58685302734375,
+      "logps/rejected": -172.87887573242188,
+      "loss": 2359.4316,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.048207301646471024,
+      "rewards/margins": 0.014712938107550144,
+      "rewards/rejected": -0.0629202350974083,
+      "step": 29
+    },
+    {
+      "epoch": 0.39522437216961714,
+      "grad_norm": 6830.68994140625,
+      "learning_rate": 3.7836854837871044e-07,
+      "logits/chosen": 0.14999902248382568,
+      "logits/rejected": 0.27044931054115295,
+      "logps/chosen": -144.25474548339844,
+      "logps/ref_chosen": -138.8655548095703,
+      "logps/ref_rejected": -169.93161010742188,
+      "logps/rejected": -176.71615600585938,
+      "loss": 2366.7632,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.05389188230037689,
+      "rewards/margins": 0.013953596353530884,
+      "rewards/rejected": -0.06784547120332718,
+      "step": 30
+    },
+    {
+      "epoch": 0.4083985179086044,
+      "grad_norm": 8562.919921875,
+      "learning_rate": 3.681721262971413e-07,
+      "logits/chosen": 0.1571384221315384,
+      "logits/rejected": 0.3112070560455322,
+      "logps/chosen": -144.13421630859375,
+      "logps/ref_chosen": -138.506103515625,
+      "logps/ref_rejected": -160.6241455078125,
+      "logps/rejected": -167.71922302246094,
+      "loss": 2360.6272,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.056280914694070816,
+      "rewards/margins": 0.014669710770249367,
+      "rewards/rejected": -0.07095062732696533,
+      "step": 31
+    },
+    {
+      "epoch": 0.4215726636475916,
+      "grad_norm": 7112.58642578125,
+      "learning_rate": 3.577159362352426e-07,
+      "logits/chosen": 0.16715647280216217,
+      "logits/rejected": 0.32336658239364624,
+      "logps/chosen": -144.36634826660156,
+      "logps/ref_chosen": -138.49879455566406,
+      "logps/ref_rejected": -174.4591827392578,
+      "logps/rejected": -181.9716033935547,
+      "loss": 2344.2522,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.05867544189095497,
+      "rewards/margins": 0.01644859090447426,
+      "rewards/rejected": -0.07512403279542923,
+      "step": 32
+    },
+    {
+      "epoch": 0.43474680938657884,
+      "grad_norm": 6419.78125,
+      "learning_rate": 3.470229631680624e-07,
+      "logits/chosen": 0.12648124992847443,
+      "logits/rejected": 0.25772759318351746,
+      "logps/chosen": -146.20086669921875,
+      "logps/ref_chosen": -140.21722412109375,
+      "logps/ref_rejected": -169.49217224121094,
+      "logps/rejected": -177.2689666748047,
+      "loss": 2331.8401,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.059836432337760925,
+      "rewards/margins": 0.017931492999196053,
+      "rewards/rejected": -0.07776791602373123,
+      "step": 33
+    },
+    {
+      "epoch": 0.44792095512556607,
+      "grad_norm": 6666.31298828125,
+      "learning_rate": 3.361167125710832e-07,
+      "logits/chosen": 0.11497840285301208,
+      "logits/rejected": 0.32681411504745483,
+      "logps/chosen": -148.2799072265625,
+      "logps/ref_chosen": -142.15771484375,
+      "logps/ref_rejected": -182.51876831054688,
+      "logps/rejected": -190.4166717529297,
+      "loss": 2333.2249,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.061221737414598465,
+      "rewards/margins": 0.017757374793291092,
+      "rewards/rejected": -0.07897911965847015,
+      "step": 34
+    },
+    {
+      "epoch": 0.4610951008645533,
+      "grad_norm": 8634.4833984375,
+      "learning_rate": 3.2502115875008516e-07,
+      "logits/chosen": 0.19959449768066406,
+      "logits/rejected": 0.3128213584423065,
+      "logps/chosen": -143.249267578125,
+      "logps/ref_chosen": -136.8671875,
+      "logps/ref_rejected": -161.7582550048828,
+      "logps/rejected": -170.11978149414062,
+      "loss": 2314.9158,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.06382085382938385,
+      "rewards/margins": 0.01979430392384529,
+      "rewards/rejected": -0.08361515402793884,
+      "step": 35
+    },
+    {
+      "epoch": 0.47426924660354053,
+      "grad_norm": 7203.060546875,
+      "learning_rate": 3.137606921404191e-07,
+      "logits/chosen": 0.22031764686107635,
+      "logits/rejected": 0.35867929458618164,
+      "logps/chosen": -147.1063690185547,
+      "logps/ref_chosen": -139.9434814453125,
+      "logps/ref_rejected": -171.1022186279297,
+      "logps/rejected": -180.0121307373047,
+      "loss": 2337.103,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.07162884622812271,
+      "rewards/margins": 0.017470190301537514,
+      "rewards/rejected": -0.08909902721643448,
+      "step": 36
+    },
+    {
+      "epoch": 0.4874433923425278,
+      "grad_norm": 9438.7119140625,
+      "learning_rate": 3.0236006569153616e-07,
+      "logits/chosen": 0.1601647287607193,
+      "logits/rejected": 0.3246475160121918,
+      "logps/chosen": -149.15200805664062,
+      "logps/ref_chosen": -142.24777221679688,
+      "logps/ref_rejected": -186.27542114257812,
+      "logps/rejected": -195.56056213378906,
+      "loss": 2276.2998,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.06904245913028717,
+      "rewards/margins": 0.023809153586626053,
+      "rewards/rejected": -0.09285160899162292,
+      "step": 37
+    },
+    {
+      "epoch": 0.500617538081515,
+      "grad_norm": 8125.13232421875,
+      "learning_rate": 2.9084434045463254e-07,
+      "logits/chosen": 0.18701249361038208,
+      "logits/rejected": 0.29296407103538513,
+      "logps/chosen": -151.26107788085938,
+      "logps/ref_chosen": -143.4994354248047,
+      "logps/ref_rejected": -169.76812744140625,
+      "logps/rejected": -179.5215606689453,
+      "loss": 2316.1897,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.07761640101671219,
+      "rewards/margins": 0.019917910918593407,
+      "rewards/rejected": -0.09753430634737015,
+      "step": 38
+    },
+    {
+      "epoch": 0.5137916838205022,
+      "grad_norm": 7210.390625,
+      "learning_rate": 2.7923883049302066e-07,
+      "logits/chosen": 0.20329824090003967,
+      "logits/rejected": 0.3062314987182617,
+      "logps/chosen": -149.1944580078125,
+      "logps/ref_chosen": -142.20091247558594,
+      "logps/ref_rejected": -176.95993041992188,
+      "logps/rejected": -186.2585906982422,
+      "loss": 2284.1282,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.06993547081947327,
+      "rewards/margins": 0.023050837218761444,
+      "rewards/rejected": -0.09298631548881531,
+      "step": 39
+    },
+    {
+      "epoch": 0.5269658295594895,
+      "grad_norm": 7052.74951171875,
+      "learning_rate": 2.6756904723632324e-07,
+      "logits/chosen": 0.2308526635169983,
+      "logits/rejected": 0.3400764465332031,
+      "logps/chosen": -145.32469177246094,
+      "logps/ref_chosen": -137.6454620361328,
+      "logps/ref_rejected": -168.80984497070312,
+      "logps/rejected": -178.9228057861328,
+      "loss": 2273.1941,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": -0.07679218798875809,
+      "rewards/margins": 0.024337470531463623,
+      "rewards/rejected": -0.10112965852022171,
+      "step": 40
+    },
+    {
+      "epoch": 0.5401399752984768,
+      "grad_norm": 6497.931640625,
+      "learning_rate": 2.5586064340081516e-07,
+      "logits/chosen": 0.20711341500282288,
+      "logits/rejected": 0.32196125388145447,
+      "logps/chosen": -152.06080627441406,
+      "logps/ref_chosen": -144.18096923828125,
+      "logps/ref_rejected": -175.95211791992188,
+      "logps/rejected": -185.96385192871094,
+      "loss": 2300.8403,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": -0.07879841327667236,
+      "rewards/margins": 0.0213189497590065,
+      "rewards/rejected": -0.10011736303567886,
+      "step": 41
+    },
+    {
+      "epoch": 0.553314121037464,
+      "grad_norm": 6679.0595703125,
+      "learning_rate": 2.4413935659918487e-07,
+      "logits/chosen": 0.18452338874340057,
+      "logits/rejected": 0.387422651052475,
+      "logps/chosen": -145.51397705078125,
+      "logps/ref_chosen": -138.06454467773438,
+      "logps/ref_rejected": -179.8441162109375,
+      "logps/rejected": -189.7871856689453,
+      "loss": 2267.5906,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": -0.07449443638324738,
+      "rewards/margins": 0.02493644505739212,
+      "rewards/rejected": -0.09943088889122009,
+      "step": 42
+    },
+    {
+      "epoch": 0.5664882667764513,
+      "grad_norm": 5415.99658203125,
+      "learning_rate": 2.3243095276367684e-07,
+      "logits/chosen": 0.2290109246969223,
+      "logits/rejected": 0.35180214047431946,
+      "logps/chosen": -149.4269256591797,
+      "logps/ref_chosen": -141.347900390625,
+      "logps/ref_rejected": -171.0642547607422,
+      "logps/rejected": -181.90652465820312,
+      "loss": 2242.7625,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.08079031854867935,
+      "rewards/margins": 0.027632439509034157,
+      "rewards/rejected": -0.10842276364564896,
+      "step": 43
+    },
+    {
+      "epoch": 0.5796624125154385,
+      "grad_norm": 7586.189453125,
+      "learning_rate": 2.2076116950697937e-07,
+      "logits/chosen": 0.10840879380702972,
+      "logits/rejected": 0.35155922174453735,
+      "logps/chosen": -149.24771118164062,
+      "logps/ref_chosen": -141.38162231445312,
+      "logps/ref_rejected": -187.5796356201172,
+      "logps/rejected": -198.4950408935547,
+      "loss": 2216.2781,
+      "rewards/accuracies": 0.8203125,
+      "rewards/chosen": -0.07866107672452927,
+      "rewards/margins": 0.030493203550577164,
+      "rewards/rejected": -0.10915428400039673,
+      "step": 44
+    },
+    {
+      "epoch": 0.5928365582544257,
+      "grad_norm": 9534.2294921875,
+      "learning_rate": 2.091556595453674e-07,
+      "logits/chosen": 0.27076855301856995,
+      "logits/rejected": 0.4277436137199402,
+      "logps/chosen": -151.55093383789062,
+      "logps/ref_chosen": -143.77825927734375,
+      "logps/ref_rejected": -179.0214385986328,
+      "logps/rejected": -189.74734497070312,
+      "loss": 2224.4258,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": -0.07772687077522278,
+      "rewards/margins": 0.029532309621572495,
+      "rewards/rejected": -0.10725918412208557,
+      "step": 45
+    },
+    {
+      "epoch": 0.606010703993413,
+      "grad_norm": 7174.7802734375,
+      "learning_rate": 1.9763993430846392e-07,
+      "logits/chosen": 0.1256578415632248,
+      "logits/rejected": 0.3058658838272095,
+      "logps/chosen": -146.8641815185547,
+      "logps/ref_chosen": -138.49636840820312,
+      "logps/ref_rejected": -175.60797119140625,
+      "logps/rejected": -186.2463836669922,
+      "loss": 2294.2419,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.08367808163166046,
+      "rewards/margins": 0.022705810144543648,
+      "rewards/rejected": -0.10638388246297836,
+      "step": 46
+    },
+    {
+      "epoch": 0.6191848497324002,
+      "grad_norm": 7637.0185546875,
+      "learning_rate": 1.862393078595809e-07,
+      "logits/chosen": 0.19172614812850952,
+      "logits/rejected": 0.3001164197921753,
+      "logps/chosen": -152.0642852783203,
+      "logps/ref_chosen": -143.7464141845703,
+      "logps/ref_rejected": -169.4178924560547,
+      "logps/rejected": -181.17225646972656,
+      "loss": 2180.1541,
+      "rewards/accuracies": 0.8359375,
+      "rewards/chosen": -0.08317889273166656,
+      "rewards/margins": 0.03436458110809326,
+      "rewards/rejected": -0.11754348129034042,
+      "step": 47
+    },
+    {
+      "epoch": 0.6323589954713874,
+      "grad_norm": 6842.74609375,
+      "learning_rate": 1.7497884124991485e-07,
+      "logits/chosen": 0.08567388355731964,
+      "logits/rejected": 0.31849536299705505,
+      "logps/chosen": -151.8147735595703,
+      "logps/ref_chosen": -143.42369079589844,
+      "logps/ref_rejected": -192.09017944335938,
+      "logps/rejected": -203.51661682128906,
+      "loss": 2217.6252,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.08391077816486359,
+      "rewards/margins": 0.03035370446741581,
+      "rewards/rejected": -0.11426447331905365,
+      "step": 48
+    },
+    {
+      "epoch": 0.6455331412103746,
+      "grad_norm": 7464.0966796875,
+      "learning_rate": 1.6388328742891678e-07,
+      "logits/chosen": 0.14193353056907654,
+      "logits/rejected": 0.29211774468421936,
+      "logps/chosen": -147.90036010742188,
+      "logps/ref_chosen": -139.34225463867188,
+      "logps/ref_rejected": -173.71649169921875,
+      "logps/rejected": -185.2205352783203,
+      "loss": 2228.885,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.08558091521263123,
+      "rewards/margins": 0.029459496960043907,
+      "rewards/rejected": -0.11504041403532028,
+      "step": 49
+    },
+    {
+      "epoch": 0.6587072869493619,
+      "grad_norm": 8806.134765625,
+      "learning_rate": 1.5297703683193753e-07,
+      "logits/chosen": 0.16522561013698578,
+      "logits/rejected": 0.3208205997943878,
+      "logps/chosen": -150.81781005859375,
+      "logps/ref_chosen": -142.66964721679688,
+      "logps/ref_rejected": -179.97023010253906,
+      "logps/rejected": -191.62022399902344,
+      "loss": 2178.3289,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": -0.08148160576820374,
+      "rewards/margins": 0.035018205642700195,
+      "rewards/rejected": -0.11649981141090393,
+      "step": 50
+    },
+    {
+      "epoch": 0.6718814326883491,
+      "grad_norm": 5882.73486328125,
+      "learning_rate": 1.422840637647574e-07,
+      "logits/chosen": 0.12343470007181168,
+      "logits/rejected": 0.21680085361003876,
+      "logps/chosen": -154.97280883789062,
+      "logps/ref_chosen": -147.04750061035156,
+      "logps/ref_rejected": -182.6226043701172,
+      "logps/rejected": -193.77963256835938,
+      "loss": 2203.8274,
+      "rewards/accuracies": 0.8046875,
+      "rewards/chosen": -0.0792531818151474,
+      "rewards/margins": 0.03231706842780113,
+      "rewards/rejected": -0.11157024651765823,
+      "step": 51
+    },
+    {
+      "epoch": 0.6850555784273363,
+      "grad_norm": 6423.96044921875,
+      "learning_rate": 1.3182787370285865e-07,
+      "logits/chosen": 0.20253154635429382,
+      "logits/rejected": 0.351491242647171,
+      "logps/chosen": -153.34129333496094,
+      "logps/ref_chosen": -145.28207397460938,
+      "logps/ref_rejected": -186.92506408691406,
+      "logps/rejected": -198.43634033203125,
+      "loss": 2184.8164,
+      "rewards/accuracies": 0.8046875,
+      "rewards/chosen": -0.08059230446815491,
+      "rewards/margins": 0.03452064096927643,
+      "rewards/rejected": -0.11511294543743134,
+      "step": 52
+    },
+    {
+      "epoch": 0.6982297241663236,
+      "grad_norm": 7110.8720703125,
+      "learning_rate": 1.2163145162128946e-07,
+      "logits/chosen": 0.17684641480445862,
+      "logits/rejected": 0.2714047133922577,
+      "logps/chosen": -154.22564697265625,
+      "logps/ref_chosen": -144.9042205810547,
+      "logps/ref_rejected": -181.88597106933594,
+      "logps/rejected": -193.96234130859375,
+      "loss": 2248.8911,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.0932142436504364,
+      "rewards/margins": 0.0275496207177639,
+      "rewards/rejected": -0.12076384574174881,
+      "step": 53
+    },
+    {
+      "epoch": 0.7114038699053108,
+      "grad_norm": 8480.2412109375,
+      "learning_rate": 1.1171721146870014e-07,
+      "logits/chosen": 0.1637565791606903,
+      "logits/rejected": 0.34545600414276123,
+      "logps/chosen": -147.90777587890625,
+      "logps/ref_chosen": -139.37796020507812,
+      "logps/ref_rejected": -178.37158203125,
+      "logps/rejected": -190.43238830566406,
+      "loss": 2175.6248,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": -0.085298091173172,
+      "rewards/margins": 0.0353100560605526,
+      "rewards/rejected": -0.12060815095901489,
+      "step": 54
+    },
+    {
+      "epoch": 0.724578015644298,
+      "grad_norm": 6994.4423828125,
+      "learning_rate": 1.0210694689661939e-07,
+      "logits/chosen": 0.1886930912733078,
+      "logits/rejected": 0.311793714761734,
+      "logps/chosen": -147.7918701171875,
+      "logps/ref_chosen": -139.57748413085938,
+      "logps/ref_rejected": -163.42202758789062,
+      "logps/rejected": -175.12420654296875,
+      "loss": 2178.6426,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.08214367181062698,
+      "rewards/margins": 0.03487822040915489,
+      "rewards/rejected": -0.11702188849449158,
+      "step": 55
+    },
+    {
+      "epoch": 0.7377521613832853,
+      "grad_norm": 5469.5498046875,
+      "learning_rate": 9.282178335227883e-08,
+      "logits/chosen": 0.13419488072395325,
+      "logits/rejected": 0.32886138558387756,
+      "logps/chosen": -153.0216064453125,
+      "logps/ref_chosen": -143.66351318359375,
+      "logps/ref_rejected": -189.92372131347656,
+      "logps/rejected": -202.5775909423828,
+      "loss": 2202.1431,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.09358088672161102,
+      "rewards/margins": 0.0329577773809433,
+      "rewards/rejected": -0.12653866410255432,
+      "step": 56
+    },
+    {
+      "epoch": 0.7509263071222725,
+      "grad_norm": 10984.908203125,
+      "learning_rate": 8.388213164029459e-08,
+      "logits/chosen": 0.21511928737163544,
+      "logits/rejected": 0.34719422459602356,
+      "logps/chosen": -151.35519409179688,
+      "logps/ref_chosen": -142.55517578125,
+      "logps/ref_rejected": -175.55435180664062,
+      "logps/rejected": -187.24032592773438,
+      "loss": 2239.1248,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.08800003677606583,
+      "rewards/margins": 0.028859596699476242,
+      "rewards/rejected": -0.11685963720083237,
+      "step": 57
+    },
+    {
+      "epoch": 0.7641004528612598,
+      "grad_norm": 8758.0439453125,
+      "learning_rate": 7.530764305528958e-08,
+      "logits/chosen": 0.20229403674602509,
+      "logits/rejected": 0.33734947443008423,
+      "logps/chosen": -151.44970703125,
+      "logps/ref_chosen": -142.6311492919922,
+      "logps/ref_rejected": -175.78897094726562,
+      "logps/rejected": -187.89028930664062,
+      "loss": 2198.1309,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.08818576484918594,
+      "rewards/margins": 0.03282759338617325,
+      "rewards/rejected": -0.12101335823535919,
+      "step": 58
+    },
+    {
+      "epoch": 0.7772745986002471,
+      "grad_norm": 7517.16748046875,
+      "learning_rate": 6.711716618408281e-08,
+      "logits/chosen": 0.16748952865600586,
+      "logits/rejected": 0.3588869273662567,
+      "logps/chosen": -153.1207275390625,
+      "logps/ref_chosen": -144.38674926757812,
+      "logps/ref_rejected": -184.999267578125,
+      "logps/rejected": -197.11062622070312,
+      "loss": 2194.5862,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.08733989298343658,
+      "rewards/margins": 0.033773742616176605,
+      "rewards/rejected": -0.12111362814903259,
+      "step": 59
+    },
+    {
+      "epoch": 0.7904487443392343,
+      "grad_norm": 6131.287109375,
+      "learning_rate": 5.932870547240454e-08,
+      "logits/chosen": 0.2141285240650177,
+      "logits/rejected": 0.3435683250427246,
+      "logps/chosen": -152.48385620117188,
+      "logps/ref_chosen": -143.2462158203125,
+      "logps/ref_rejected": -177.8916778564453,
+      "logps/rejected": -190.52423095703125,
+      "loss": 2187.7061,
+      "rewards/accuracies": 0.8046875,
+      "rewards/chosen": -0.09237637370824814,
+      "rewards/margins": 0.033949077129364014,
+      "rewards/rejected": -0.12632544338703156,
+      "step": 60
+    },
+    {
+      "epoch": 0.8036228900782215,
+      "grad_norm": 8281.1650390625,
+      "learning_rate": 5.1959381647217665e-08,
+      "logits/chosen": 0.1745128035545349,
+      "logits/rejected": 0.3122703731060028,
+      "logps/chosen": -152.68939208984375,
+      "logps/ref_chosen": -143.92420959472656,
+      "logps/ref_rejected": -176.6068115234375,
+      "logps/rejected": -188.49099731445312,
+      "loss": 2219.0474,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.08765168488025665,
+      "rewards/margins": 0.031190264970064163,
+      "rewards/rejected": -0.11884195357561111,
+      "step": 61
+    },
+    {
+      "epoch": 0.8167970358172087,
+      "grad_norm": 5113.0146484375,
+      "learning_rate": 4.502539408164385e-08,
+      "logits/chosen": 0.13161124289035797,
+      "logits/rejected": 0.2597178816795349,
+      "logps/chosen": -149.14198303222656,
+      "logps/ref_chosen": -140.12322998046875,
+      "logps/ref_rejected": -171.29669189453125,
+      "logps/rejected": -183.97909545898438,
+      "loss": 2163.1401,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.09018754214048386,
+      "rewards/margins": 0.03663650527596474,
+      "rewards/rejected": -0.1268240511417389,
+      "step": 62
+    },
+    {
+      "epoch": 0.829971181556196,
+      "grad_norm": 7166.9453125,
+      "learning_rate": 3.854198518522564e-08,
+      "logits/chosen": 0.22974729537963867,
+      "logits/rejected": 0.36869755387306213,
+      "logps/chosen": -145.85195922851562,
+      "logps/ref_chosen": -136.71630859375,
+      "logps/ref_rejected": -171.1470947265625,
+      "logps/rejected": -183.384765625,
+      "loss": 2217.2756,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.09135645627975464,
+      "rewards/margins": 0.031020279973745346,
+      "rewards/rejected": -0.12237672507762909,
+      "step": 63
+    },
+    {
+      "epoch": 0.8431453272951832,
+      "grad_norm": 7681.93994140625,
+      "learning_rate": 3.2523406897802444e-08,
+      "logits/chosen": 0.23416659235954285,
+      "logits/rejected": 0.36190101504325867,
+      "logps/chosen": -154.23883056640625,
+      "logps/ref_chosen": -144.72784423828125,
+      "logps/ref_rejected": -181.54769897460938,
+      "logps/rejected": -194.52224731445312,
+      "loss": 2186.8882,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.09511007368564606,
+      "rewards/margins": 0.03463532403111458,
+      "rewards/rejected": -0.12974540889263153,
+      "step": 64
+    },
+    {
+      "epoch": 0.8563194730341704,
+      "grad_norm": 7177.78369140625,
+      "learning_rate": 2.6982889360653376e-08,
+      "logits/chosen": 0.17723366618156433,
+      "logits/rejected": 0.3242289125919342,
+      "logps/chosen": -151.72543334960938,
+      "logps/ref_chosen": -142.64617919921875,
+      "logps/ref_rejected": -174.40614318847656,
+      "logps/rejected": -186.78662109375,
+      "loss": 2196.0581,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.09079259634017944,
+      "rewards/margins": 0.033012229949235916,
+      "rewards/rejected": -0.12380482256412506,
+      "step": 65
+    },
+    {
+      "epoch": 0.8694936187731577,
+      "grad_norm": 8175.220703125,
+      "learning_rate": 2.1932611833775843e-08,
+      "logits/chosen": 0.08193757385015488,
+      "logits/rejected": 0.26095396280288696,
+      "logps/chosen": -147.0934295654297,
+      "logps/ref_chosen": -138.63568115234375,
+      "logps/ref_rejected": -182.35244750976562,
+      "logps/rejected": -194.43313598632812,
+      "loss": 2168.0254,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.08457762002944946,
+      "rewards/margins": 0.03622942045331001,
+      "rewards/rejected": -0.12080704420804977,
+      "step": 66
+    },
+    {
+      "epoch": 0.8826677645121449,
+      "grad_norm": 7215.3154296875,
+      "learning_rate": 1.738367592322837e-08,
+      "logits/chosen": 0.1766711324453354,
+      "logits/rejected": 0.3498009741306305,
+      "logps/chosen": -150.53700256347656,
+      "logps/ref_chosen": -141.5865478515625,
+      "logps/ref_rejected": -185.27081298828125,
+      "logps/rejected": -198.028564453125,
+      "loss": 2155.2078,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.0895046666264534,
+      "rewards/margins": 0.03807291388511658,
+      "rewards/rejected": -0.12757760286331177,
+      "step": 67
+    },
+    {
+      "epoch": 0.8958419102511321,
+      "grad_norm": 8997.7958984375,
+      "learning_rate": 1.3346081177391472e-08,
+      "logits/chosen": 0.14675183594226837,
+      "logits/rejected": 0.3305708169937134,
+      "logps/chosen": -148.1304931640625,
+      "logps/ref_chosen": -139.1151580810547,
+      "logps/ref_rejected": -176.0420684814453,
+      "logps/rejected": -188.79638671875,
+      "loss": 2163.0139,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.09015330672264099,
+      "rewards/margins": 0.03738997504115105,
+      "rewards/rejected": -0.12754328548908234,
+      "step": 68
+    },
+    {
+      "epoch": 0.9090160559901194,
+      "grad_norm": 6634.298828125,
+      "learning_rate": 9.828703105789981e-09,
+      "logits/chosen": 0.11062157154083252,
+      "logits/rejected": 0.26717525720596313,
+      "logps/chosen": -157.52674865722656,
+      "logps/ref_chosen": -148.0703125,
+      "logps/ref_rejected": -185.28785705566406,
+      "logps/rejected": -197.8096923828125,
+      "loss": 2219.3191,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.09456441551446915,
+      "rewards/margins": 0.030653683468699455,
+      "rewards/rejected": -0.12521810829639435,
+      "step": 69
+    },
+    {
+      "epoch": 0.9221902017291066,
+      "grad_norm": 6588.52099609375,
+      "learning_rate": 6.839273668796747e-09,
+      "logits/chosen": 0.20371803641319275,
+      "logits/rejected": 0.38707682490348816,
+      "logps/chosen": -150.7587127685547,
+      "logps/ref_chosen": -142.1817169189453,
+      "logps/ref_rejected": -183.52871704101562,
+      "logps/rejected": -195.626953125,
+      "loss": 2180.2317,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.08577002584934235,
+      "rewards/margins": 0.03521224111318588,
+      "rewards/rejected": -0.12098225951194763,
+      "step": 70
+    },
+    {
+      "epoch": 0.9353643474680938,
+      "grad_norm": 7922.1845703125,
+      "learning_rate": 4.384364281105973e-09,
+      "logits/chosen": 0.14600388705730438,
+      "logits/rejected": 0.3577485978603363,
+      "logps/chosen": -151.2028045654297,
+      "logps/ref_chosen": -142.00233459472656,
+      "logps/ref_rejected": -188.67965698242188,
+      "logps/rejected": -201.70025634765625,
+      "loss": 2152.5889,
+      "rewards/accuracies": 0.7890625,
+      "rewards/chosen": -0.09200464934110641,
+      "rewards/margins": 0.038201265037059784,
+      "rewards/rejected": -0.1302059143781662,
+      "step": 71
+    },
+    {
+      "epoch": 0.9485384932070811,
+      "grad_norm": 6042.76123046875,
+      "learning_rate": 2.469371366337264e-09,
+      "logits/chosen": 0.18784670531749725,
+      "logits/rejected": 0.33177098631858826,
+      "logps/chosen": -151.64651489257812,
+      "logps/ref_chosen": -142.16261291503906,
+      "logps/ref_rejected": -195.17788696289062,
+      "logps/rejected": -207.6988983154297,
+      "loss": 2222.146,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": -0.09483905881643295,
+      "rewards/margins": 0.030371030792593956,
+      "rewards/rejected": -0.12521009147167206,
+      "step": 72
+    },
+    {
+      "epoch": 0.9617126389460683,
+      "grad_norm": 5766.6953125,
+      "learning_rate": 1.0985044945254762e-09,
+      "logits/chosen": 0.13510531187057495,
+      "logits/rejected": 0.3228178024291992,
+      "logps/chosen": -149.89263916015625,
+      "logps/ref_chosen": -140.62818908691406,
+      "logps/ref_rejected": -181.2576141357422,
+      "logps/rejected": -194.0572509765625,
+      "loss": 2178.6985,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.09264436364173889,
+      "rewards/margins": 0.035351961851119995,
+      "rewards/rejected": -0.12799634039402008,
+      "step": 73
+    },
+    {
+      "epoch": 0.9748867846850556,
+      "grad_norm": 8777.2763671875,
+      "learning_rate": 2.7477712857215675e-10,
+      "logits/chosen": 0.14389634132385254,
+      "logits/rejected": 0.3162282109260559,
+      "logps/chosen": -151.93350219726562,
+      "logps/ref_chosen": -143.0329132080078,
+      "logps/ref_rejected": -187.8455810546875,
+      "logps/rejected": -200.83340454101562,
+      "loss": 2124.8625,
+      "rewards/accuracies": 0.8515625,
+      "rewards/chosen": -0.08900578320026398,
+      "rewards/margins": 0.040872570127248764,
+      "rewards/rejected": -0.12987834215164185,
+      "step": 74
+    },
+    {
+      "epoch": 0.9880609304240429,
+      "grad_norm": 6027.6533203125,
+      "learning_rate": 0.0,
+      "logits/chosen": 0.14975285530090332,
+      "logits/rejected": 0.2791685461997986,
+      "logps/chosen": -151.98268127441406,
+      "logps/ref_chosen": -142.6675262451172,
+      "logps/ref_rejected": -172.9785614013672,
+      "logps/rejected": -185.24148559570312,
+      "loss": 2232.3572,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.09315158426761627,
+      "rewards/margins": 0.029477719217538834,
+      "rewards/rejected": -0.12262929230928421,
+      "step": 75
+    },
+    {
+      "epoch": 0.9880609304240429,
+      "step": 75,
+      "total_flos": 0.0,
+      "train_loss": 2318.3018196614585,
+      "train_runtime": 5362.5076,
+      "train_samples_per_second": 1.812,
+      "train_steps_per_second": 0.014
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 75,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 64,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ea31e2520a407dfedcf990d30d6e5ea37a809e00c66520b0daccabcc98bad11
+size 7352

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1_epoch3/README.md

@@ -0,0 +1,59 @@
+---
+library_name: transformers
+base_model: /home/v-liuzichuan/huggingface/Llama-3.2-3B-Instruct
+tags:
+- alignment-handbook
+- generated_from_trainer
+datasets:
+- /home/v-liuzichuan/huggingface/princeton-nlp/llama3-math
+model-index:
+- name: Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1_epoch3
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1_epoch3
+
+This model is a fine-tuned version of [/home/v-liuzichuan/huggingface/Llama-3.2-3B-Instruct](https://huggingface.co//home/v-liuzichuan/huggingface/Llama-3.2-3B-Instruct) on the /home/v-liuzichuan/huggingface/princeton-nlp/llama3-math dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- gradient_accumulation_steps: 64
+- total_train_batch_size: 128
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.45.2
+- Pytorch 2.5.1+cu121
+- Datasets 3.5.0
+- Tokenizers 0.20.3

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1_epoch3/all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9969120977149523,
+    "total_flos": 0.0,
+    "train_loss": 1752.3481854006056,
+    "train_runtime": 13961.4025,
+    "train_samples": 29145,
+    "train_samples_per_second": 2.088,
+    "train_steps_per_second": 0.016
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1_epoch3/config.json

@@ -0,0 +1,40 @@
+{
+  "_name_or_path": "/home/v-liuzichuan/huggingface/Llama-3.2-3B-Instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.2",
+  "use_cache": true,
+  "vocab_size": 128256
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1_epoch3/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.45.2"
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1_epoch3/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fa267e0c272699d33e3c070b9c97b3382601e63c6fd0f4084897b51d094a0ec
+size 4965799096

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1_epoch3/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a1e6860342e9724eb4d182ffb3d8ad250457cc1f6e46b3627215784440328b0
+size 2247734992

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1_epoch3/model.safetensors.index.json

@@ -0,0 +1,262 @@
+{
+  "metadata": {
+    "total_size": 7213504512
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1_epoch3/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|eot_id|>"
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1_epoch3/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1_epoch3/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- if strftime_now is defined %}\n        {%- set date_string = strftime_now(\"%d %b %Y\") %}\n    {%- else %}\n        {%- set date_string = \"26 Jul 2024\" %}\n    {%- endif %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n        {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n        {{- '\"parameters\": ' }}\n        {{- tool_call.arguments | tojson }}\n        {{- \"}\" }}\n        {{- \"<|eot_id|>\" }}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 2048,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1_epoch3/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9969120977149523,
+    "total_flos": 0.0,
+    "train_loss": 1752.3481854006056,
+    "train_runtime": 13961.4025,
+    "train_samples": 29145,
+    "train_samples_per_second": 2.088,
+    "train_steps_per_second": 0.016
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk1_epoch3/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72f9eefd5644f3e7b04a1fb2cb27d27728da16ee4e5ab23ebbe27b18bb9e5302
+size 7352

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/README.md

@@ -0,0 +1,59 @@
+---
+library_name: transformers
+base_model: /home/v-liuzichuan/huggingface/Llama-3.2-3B-Instruct
+tags:
+- alignment-handbook
+- generated_from_trainer
+datasets:
+- /home/v-liuzichuan/huggingface/princeton-nlp/llama3-math
+model-index:
+- name: '2'
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# 2
+
+This model is a fine-tuned version of [/home/v-liuzichuan/huggingface/Llama-3.2-3B-Instruct](https://huggingface.co//home/v-liuzichuan/huggingface/Llama-3.2-3B-Instruct) on the /home/v-liuzichuan/huggingface/princeton-nlp/llama3-math dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- gradient_accumulation_steps: 64
+- total_train_batch_size: 128
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.45.2
+- Pytorch 2.5.1+cu121
+- Datasets 3.5.0
+- Tokenizers 0.20.3

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9880609304240429,
+    "total_flos": 0.0,
+    "train_loss": 1284.6718611653646,
+    "train_runtime": 5197.2653,
+    "train_samples": 9715,
+    "train_samples_per_second": 1.869,
+    "train_steps_per_second": 0.014
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/config.json

@@ -0,0 +1,40 @@
+{
+  "_name_or_path": "/home/v-liuzichuan/dpo/outputs_ipo//temp_path",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.2",
+  "use_cache": true,
+  "vocab_size": 128256
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.45.2"
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a9abeae06bf7f918b6ca8011797790ab30a4c85410abb0e841a0a31df227c3c
+size 4965799096

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b654f33377137c4ff410e5ff4844fbf6db90b7013eda592e5d8992e1e3214774
+size 2247734992

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/model.safetensors.index.json

@@ -0,0 +1,262 @@
+{
+  "metadata": {
+    "total_size": 7213504512
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/raw_datasets.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d89ef369c3a210b3f0617ba98bcbe2181b12d114b10eaf1ca89f1b55c7000130
+size 14697198

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|eot_id|>"
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- if strftime_now is defined %}\n        {%- set date_string = strftime_now(\"%d %b %Y\") %}\n    {%- else %}\n        {%- set date_string = \"26 Jul 2024\" %}\n    {%- endif %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n        {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n        {{- '\"parameters\": ' }}\n        {{- tool_call.arguments | tojson }}\n        {{- \"}\" }}\n        {{- \"<|eot_id|>\" }}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 2048,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9880609304240429,
+    "total_flos": 0.0,
+    "train_loss": 1284.6718611653646,
+    "train_runtime": 5197.2653,
+    "train_samples": 9715,
+    "train_samples_per_second": 1.869,
+    "train_steps_per_second": 0.014
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/trainer_state.json

@@ -0,0 +1,1392 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9880609304240429,
+  "eval_steps": 500,
+  "global_step": 75,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.013174145738987238,
+      "grad_norm": 23949.7109375,
+      "learning_rate": 6.25e-08,
+      "logits/chosen": -0.8008705973625183,
+      "logits/rejected": -0.8651990294456482,
+      "logps/chosen": -250.500244140625,
+      "logps/ref_chosen": -254.47015380859375,
+      "logps/ref_rejected": -418.95074462890625,
+      "logps/rejected": -411.23785400390625,
+      "loss": 1468.7465,
+      "rewards/accuracies": 0.3046875,
+      "rewards/chosen": 0.03969910740852356,
+      "rewards/margins": -0.03742925450205803,
+      "rewards/rejected": 0.0771283507347107,
+      "sft_loss": 0.9547704458236694,
+      "step": 1
+    },
+    {
+      "epoch": 0.026348291477974475,
+      "grad_norm": 42517.4296875,
+      "learning_rate": 1.25e-07,
+      "logits/chosen": -0.8648843765258789,
+      "logits/rejected": -0.8835014700889587,
+      "logps/chosen": -271.94342041015625,
+      "logps/ref_chosen": -276.519775390625,
+      "logps/ref_rejected": -435.0823669433594,
+      "logps/rejected": -427.2591857910156,
+      "loss": 1440.641,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": 0.045763421803712845,
+      "rewards/margins": -0.03246838226914406,
+      "rewards/rejected": 0.0782318040728569,
+      "sft_loss": 0.9081498980522156,
+      "step": 2
+    },
+    {
+      "epoch": 0.03952243721696171,
+      "grad_norm": 45039.2109375,
+      "learning_rate": 1.875e-07,
+      "logits/chosen": -0.7890776991844177,
+      "logits/rejected": -0.8727779984474182,
+      "logps/chosen": -253.0457763671875,
+      "logps/ref_chosen": -257.1988220214844,
+      "logps/ref_rejected": -393.6779479980469,
+      "logps/rejected": -387.13726806640625,
+      "loss": 1390.0684,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": 0.04153094068169594,
+      "rewards/margins": -0.023875586688518524,
+      "rewards/rejected": 0.06540653109550476,
+      "sft_loss": 0.8998649716377258,
+      "step": 3
+    },
+    {
+      "epoch": 0.05269658295594895,
+      "grad_norm": 40517.47265625,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": -0.7816249132156372,
+      "logits/rejected": -0.8107084035873413,
+      "logps/chosen": -260.0189208984375,
+      "logps/ref_chosen": -264.57086181640625,
+      "logps/ref_rejected": -404.1727600097656,
+      "logps/rejected": -396.4281005859375,
+      "loss": 1439.671,
+      "rewards/accuracies": 0.3046875,
+      "rewards/chosen": 0.04551965743303299,
+      "rewards/margins": -0.03192698955535889,
+      "rewards/rejected": 0.07744664698839188,
+      "sft_loss": 0.9242280721664429,
+      "step": 4
+    },
+    {
+      "epoch": 0.06587072869493618,
+      "grad_norm": 40508.13671875,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -0.7920335531234741,
+      "logits/rejected": -0.8334716558456421,
+      "logps/chosen": -257.8485107421875,
+      "logps/ref_chosen": -261.6973571777344,
+      "logps/ref_rejected": -415.48101806640625,
+      "logps/rejected": -408.35888671875,
+      "loss": 1439.3862,
+      "rewards/accuracies": 0.2578125,
+      "rewards/chosen": 0.03848864510655403,
+      "rewards/margins": -0.0327322855591774,
+      "rewards/rejected": 0.07122092694044113,
+      "sft_loss": 0.9617606401443481,
+      "step": 5
+    },
+    {
+      "epoch": 0.07904487443392343,
+      "grad_norm": 67280.8203125,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": -0.8593892455101013,
+      "logits/rejected": -0.9203681349754333,
+      "logps/chosen": -250.87802124023438,
+      "logps/ref_chosen": -254.50120544433594,
+      "logps/ref_rejected": -415.44366455078125,
+      "logps/rejected": -408.802734375,
+      "loss": 1426.8674,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": 0.03623175993561745,
+      "rewards/margins": -0.030177462846040726,
+      "rewards/rejected": 0.06640923023223877,
+      "sft_loss": 0.8602447509765625,
+      "step": 6
+    },
+    {
+      "epoch": 0.09221902017291066,
+      "grad_norm": 36309.33984375,
+      "learning_rate": 4.375e-07,
+      "logits/chosen": -0.8027951121330261,
+      "logits/rejected": -0.8602517247200012,
+      "logps/chosen": -253.07276916503906,
+      "logps/ref_chosen": -256.30010986328125,
+      "logps/ref_rejected": -417.3987731933594,
+      "logps/rejected": -413.0194396972656,
+      "loss": 1319.4021,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.03227325156331062,
+      "rewards/margins": -0.011520068161189556,
+      "rewards/rejected": 0.043793320655822754,
+      "sft_loss": 0.9406993985176086,
+      "step": 7
+    },
+    {
+      "epoch": 0.1053931659118979,
+      "grad_norm": 58247.61328125,
+      "learning_rate": 5e-07,
+      "logits/chosen": -0.8113671541213989,
+      "logits/rejected": -0.8746984004974365,
+      "logps/chosen": -261.3500061035156,
+      "logps/ref_chosen": -262.356201171875,
+      "logps/ref_rejected": -429.6075439453125,
+      "logps/rejected": -429.34735107421875,
+      "loss": 1226.9708,
+      "rewards/accuracies": 0.484375,
+      "rewards/chosen": 0.010061981156468391,
+      "rewards/margins": 0.007459938060492277,
+      "rewards/rejected": 0.002602042630314827,
+      "sft_loss": 0.925779402256012,
+      "step": 8
+    },
+    {
+      "epoch": 0.11856731165088513,
+      "grad_norm": 35830.9296875,
+      "learning_rate": 4.997252228714278e-07,
+      "logits/chosen": -0.7887070178985596,
+      "logits/rejected": -0.8501583933830261,
+      "logps/chosen": -247.213623046875,
+      "logps/ref_chosen": -246.46566772460938,
+      "logps/ref_rejected": -401.689208984375,
+      "logps/rejected": -405.4938659667969,
+      "loss": 1137.1165,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.007479269523173571,
+      "rewards/margins": 0.030567554756999016,
+      "rewards/rejected": -0.03804682567715645,
+      "sft_loss": 0.9024340510368347,
+      "step": 9
+    },
+    {
+      "epoch": 0.13174145738987236,
+      "grad_norm": 49081.078125,
+      "learning_rate": 4.989014955054745e-07,
+      "logits/chosen": -0.7538288831710815,
+      "logits/rejected": -0.8280922174453735,
+      "logps/chosen": -260.5111389160156,
+      "logps/ref_chosen": -255.78009033203125,
+      "logps/ref_rejected": -355.2666015625,
+      "logps/rejected": -364.0128173828125,
+      "loss": 1120.6088,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.047310683876276016,
+      "rewards/margins": 0.04015128314495087,
+      "rewards/rejected": -0.08746197074651718,
+      "sft_loss": 0.9213616847991943,
+      "step": 10
+    },
+    {
+      "epoch": 0.14491560312885962,
+      "grad_norm": 27151.37109375,
+      "learning_rate": 4.975306286336627e-07,
+      "logits/chosen": -0.8138033747673035,
+      "logits/rejected": -0.8641115427017212,
+      "logps/chosen": -279.50128173828125,
+      "logps/ref_chosen": -268.767578125,
+      "logps/ref_rejected": -417.3045654296875,
+      "logps/rejected": -440.1104431152344,
+      "loss": 1080.8665,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.10733672231435776,
+      "rewards/margins": 0.12072216719388962,
+      "rewards/rejected": -0.22805888950824738,
+      "sft_loss": 0.9601907730102539,
+      "step": 11
+    },
+    {
+      "epoch": 0.15808974886784685,
+      "grad_norm": 35450.01953125,
+      "learning_rate": 4.956156357188939e-07,
+      "logits/chosen": -0.7551673650741577,
+      "logits/rejected": -0.8324231505393982,
+      "logps/chosen": -274.9906921386719,
+      "logps/ref_chosen": -261.2640380859375,
+      "logps/ref_rejected": -356.03948974609375,
+      "logps/rejected": -378.89349365234375,
+      "loss": 1123.4912,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.1372665911912918,
+      "rewards/margins": 0.09127368032932281,
+      "rewards/rejected": -0.22854028642177582,
+      "sft_loss": 1.0183186531066895,
+      "step": 12
+    },
+    {
+      "epoch": 0.17126389460683408,
+      "grad_norm": 31516.70703125,
+      "learning_rate": 4.931607263312032e-07,
+      "logits/chosen": -0.7455427050590515,
+      "logits/rejected": -0.816511332988739,
+      "logps/chosen": -289.495361328125,
+      "logps/ref_chosen": -272.1609191894531,
+      "logps/ref_rejected": -417.3423767089844,
+      "logps/rejected": -454.0290222167969,
+      "loss": 1218.0366,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.17334474623203278,
+      "rewards/margins": 0.19352208077907562,
+      "rewards/rejected": -0.366866797208786,
+      "sft_loss": 1.0172396898269653,
+      "step": 13
+    },
+    {
+      "epoch": 0.1844380403458213,
+      "grad_norm": 22413.31640625,
+      "learning_rate": 4.9017129689421e-07,
+      "logits/chosen": -0.8126018643379211,
+      "logits/rejected": -0.8541157245635986,
+      "logps/chosen": -272.24163818359375,
+      "logps/ref_chosen": -254.89801025390625,
+      "logps/ref_rejected": -403.5438537597656,
+      "logps/rejected": -441.9024963378906,
+      "loss": 1253.6353,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.17343664169311523,
+      "rewards/margins": 0.21014949679374695,
+      "rewards/rejected": -0.38358616828918457,
+      "sft_loss": 0.9973514080047607,
+      "step": 14
+    },
+    {
+      "epoch": 0.19761218608480857,
+      "grad_norm": 299858.125,
+      "learning_rate": 4.866539188226085e-07,
+      "logits/chosen": -0.821877658367157,
+      "logits/rejected": -0.8447117805480957,
+      "logps/chosen": -270.3739318847656,
+      "logps/ref_chosen": -250.36923217773438,
+      "logps/ref_rejected": -402.3016662597656,
+      "logps/rejected": -448.3406982421875,
+      "loss": 2166.4175,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.2000468224287033,
+      "rewards/margins": 0.2603432536125183,
+      "rewards/rejected": -0.4603900611400604,
+      "sft_loss": 0.9630093574523926,
+      "step": 15
+    },
+    {
+      "epoch": 0.2107863318237958,
+      "grad_norm": 20275.111328125,
+      "learning_rate": 4.826163240767716e-07,
+      "logits/chosen": -0.7847133874893188,
+      "logits/rejected": -0.8517911434173584,
+      "logps/chosen": -274.80609130859375,
+      "logps/ref_chosen": -253.88784790039062,
+      "logps/ref_rejected": -363.81549072265625,
+      "logps/rejected": -404.54119873046875,
+      "loss": 1400.3004,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.2091825008392334,
+      "rewards/margins": 0.198074609041214,
+      "rewards/rejected": -0.4072571396827698,
+      "sft_loss": 0.9957829713821411,
+      "step": 16
+    },
+    {
+      "epoch": 0.22396047756278303,
+      "grad_norm": 144756.109375,
+      "learning_rate": 4.780673881662242e-07,
+      "logits/chosen": -0.7692462801933289,
+      "logits/rejected": -0.750064492225647,
+      "logps/chosen": -312.2320861816406,
+      "logps/ref_chosen": -280.298095703125,
+      "logps/ref_rejected": -426.6932373046875,
+      "logps/rejected": -485.9967041015625,
+      "loss": 2140.3831,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.3193399906158447,
+      "rewards/margins": 0.27369457483291626,
+      "rewards/rejected": -0.5930346250534058,
+      "sft_loss": 1.019205093383789,
+      "step": 17
+    },
+    {
+      "epoch": 0.23713462330177026,
+      "grad_norm": 147186.65625,
+      "learning_rate": 4.730171106393466e-07,
+      "logits/chosen": -0.8014205694198608,
+      "logits/rejected": -0.8366638422012329,
+      "logps/chosen": -272.2122497558594,
+      "logps/ref_chosen": -245.93484497070312,
+      "logps/ref_rejected": -355.2913818359375,
+      "logps/rejected": -402.57916259765625,
+      "loss": 1260.5695,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.26277416944503784,
+      "rewards/margins": 0.21010375022888184,
+      "rewards/rejected": -0.4728779196739197,
+      "sft_loss": 1.0054181814193726,
+      "step": 18
+    },
+    {
+      "epoch": 0.2503087690407575,
+      "grad_norm": 9353.1171875,
+      "learning_rate": 4.6747659310219757e-07,
+      "logits/chosen": -0.7688182592391968,
+      "logits/rejected": -0.8143107891082764,
+      "logps/chosen": -313.2867126464844,
+      "logps/ref_chosen": -282.3853759765625,
+      "logps/ref_rejected": -423.46612548828125,
+      "logps/rejected": -485.9477844238281,
+      "loss": 2975.3206,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.3090134561061859,
+      "rewards/margins": 0.3158027231693268,
+      "rewards/rejected": -0.6248162388801575,
+      "sft_loss": 1.0559428930282593,
+      "step": 19
+    },
+    {
+      "epoch": 0.2634829147797447,
+      "grad_norm": 71948.1328125,
+      "learning_rate": 4.6145801481477433e-07,
+      "logits/chosen": -0.7672979831695557,
+      "logits/rejected": -0.8246533274650574,
+      "logps/chosen": -278.12701416015625,
+      "logps/ref_chosen": -250.12939453125,
+      "logps/ref_rejected": -400.04193115234375,
+      "logps/rejected": -459.34783935546875,
+      "loss": 1524.6759,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.2799761891365051,
+      "rewards/margins": 0.31308287382125854,
+      "rewards/rejected": -0.5930590629577637,
+      "sft_loss": 0.995878279209137,
+      "step": 20
+    },
+    {
+      "epoch": 0.276657060518732,
+      "grad_norm": 19010.67578125,
+      "learning_rate": 4.549746059183561e-07,
+      "logits/chosen": -0.7679699063301086,
+      "logits/rejected": -0.7934538125991821,
+      "logps/chosen": -290.7159729003906,
+      "logps/ref_chosen": -260.5438232421875,
+      "logps/ref_rejected": -394.02606201171875,
+      "logps/rejected": -455.3935546875,
+      "loss": 2176.1531,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.3017215132713318,
+      "rewards/margins": 0.31195297837257385,
+      "rewards/rejected": -0.6136744618415833,
+      "sft_loss": 1.0407434701919556,
+      "step": 21
+    },
+    {
+      "epoch": 0.28983120625771924,
+      "grad_norm": 74092.6953125,
+      "learning_rate": 4.480406183527823e-07,
+      "logits/chosen": -0.8038030862808228,
+      "logits/rejected": -0.828942060470581,
+      "logps/chosen": -294.260009765625,
+      "logps/ref_chosen": -264.4549560546875,
+      "logps/ref_rejected": -426.1365661621094,
+      "logps/rejected": -491.14739990234375,
+      "loss": 2098.3057,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.29805096983909607,
+      "rewards/margins": 0.3520573675632477,
+      "rewards/rejected": -0.6501083970069885,
+      "sft_loss": 1.0383522510528564,
+      "step": 22
+    },
+    {
+      "epoch": 0.3030053519967065,
+      "grad_norm": 23798.248046875,
+      "learning_rate": 4.4067129452759546e-07,
+      "logits/chosen": -0.8293471932411194,
+      "logits/rejected": -0.8665866851806641,
+      "logps/chosen": -276.66107177734375,
+      "logps/ref_chosen": -248.43484497070312,
+      "logps/ref_rejected": -354.51226806640625,
+      "logps/rejected": -406.6967468261719,
+      "loss": 1895.6882,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.2822622060775757,
+      "rewards/margins": 0.2395820915699005,
+      "rewards/rejected": -0.5218443274497986,
+      "sft_loss": 1.0060269832611084,
+      "step": 23
+    },
+    {
+      "epoch": 0.3161794977356937,
+      "grad_norm": 85321.9296875,
+      "learning_rate": 4.3288283381591725e-07,
+      "logits/chosen": -0.8108458518981934,
+      "logits/rejected": -0.8491148352622986,
+      "logps/chosen": -268.555419921875,
+      "logps/ref_chosen": -241.54579162597656,
+      "logps/ref_rejected": -381.7640686035156,
+      "logps/rejected": -436.7099914550781,
+      "loss": 2128.0498,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.27009618282318115,
+      "rewards/margins": 0.27936306595802307,
+      "rewards/rejected": -0.5494592189788818,
+      "sft_loss": 0.9779033660888672,
+      "step": 24
+    },
+    {
+      "epoch": 0.32935364347468093,
+      "grad_norm": 26374.453125,
+      "learning_rate": 4.246923569447104e-07,
+      "logits/chosen": -0.7603333592414856,
+      "logits/rejected": -0.7976529002189636,
+      "logps/chosen": -289.4198913574219,
+      "logps/ref_chosen": -263.8042297363281,
+      "logps/ref_rejected": -432.7107238769531,
+      "logps/rejected": -491.9762878417969,
+      "loss": 2442.6104,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.2561565637588501,
+      "rewards/margins": 0.3364991247653961,
+      "rewards/rejected": -0.5926557183265686,
+      "sft_loss": 1.008938193321228,
+      "step": 25
+    },
+    {
+      "epoch": 0.34252778921366817,
+      "grad_norm": 18185.58203125,
+      "learning_rate": 4.161178683597054e-07,
+      "logits/chosen": -0.7645914554595947,
+      "logits/rejected": -0.7908961176872253,
+      "logps/chosen": -283.41595458984375,
+      "logps/ref_chosen": -258.4693603515625,
+      "logps/ref_rejected": -396.6169128417969,
+      "logps/rejected": -451.3160400390625,
+      "loss": 1601.3,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2494659125804901,
+      "rewards/margins": 0.29752564430236816,
+      "rewards/rejected": -0.5469915866851807,
+      "sft_loss": 1.0247998237609863,
+      "step": 26
+    },
+    {
+      "epoch": 0.3557019349526554,
+      "grad_norm": 70844.5859375,
+      "learning_rate": 4.0717821664772124e-07,
+      "logits/chosen": -0.8039371967315674,
+      "logits/rejected": -0.8496546149253845,
+      "logps/chosen": -265.72076416015625,
+      "logps/ref_chosen": -243.9918670654297,
+      "logps/ref_rejected": -415.019775390625,
+      "logps/rejected": -469.927490234375,
+      "loss": 1508.8497,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.21728886663913727,
+      "rewards/margins": 0.3317885100841522,
+      "rewards/rejected": -0.5490773320198059,
+      "sft_loss": 0.9742772579193115,
+      "step": 27
+    },
+    {
+      "epoch": 0.3688760806916426,
+      "grad_norm": 17698.689453125,
+      "learning_rate": 3.978930531033806e-07,
+      "logits/chosen": -0.8012359142303467,
+      "logits/rejected": -0.8333377242088318,
+      "logps/chosen": -286.1438903808594,
+      "logps/ref_chosen": -262.3642578125,
+      "logps/ref_rejected": -404.7378845214844,
+      "logps/rejected": -453.0112609863281,
+      "loss": 1529.8362,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.2377963364124298,
+      "rewards/margins": 0.24493779242038727,
+      "rewards/rejected": -0.48273414373397827,
+      "sft_loss": 1.0474350452423096,
+      "step": 28
+    },
+    {
+      "epoch": 0.3820502264306299,
+      "grad_norm": 31423.068359375,
+      "learning_rate": 3.882827885312998e-07,
+      "logits/chosen": -0.8194433450698853,
+      "logits/rejected": -0.8658930659294128,
+      "logps/chosen": -269.17816162109375,
+      "logps/ref_chosen": -248.24476623535156,
+      "logps/ref_rejected": -385.9969787597656,
+      "logps/rejected": -430.42547607421875,
+      "loss": 1514.3403,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.20933374762535095,
+      "rewards/margins": 0.2349512279033661,
+      "rewards/rejected": -0.44428494572639465,
+      "sft_loss": 0.9792473316192627,
+      "step": 29
+    },
+    {
+      "epoch": 0.39522437216961714,
+      "grad_norm": 21664.841796875,
+      "learning_rate": 3.7836854837871044e-07,
+      "logits/chosen": -0.7560093998908997,
+      "logits/rejected": -0.8075149655342102,
+      "logps/chosen": -274.10748291015625,
+      "logps/ref_chosen": -252.64498901367188,
+      "logps/ref_rejected": -389.426025390625,
+      "logps/rejected": -431.0108642578125,
+      "loss": 1284.9393,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.21462516486644745,
+      "rewards/margins": 0.20122365653514862,
+      "rewards/rejected": -0.4158487915992737,
+      "sft_loss": 0.9510519504547119,
+      "step": 30
+    },
+    {
+      "epoch": 0.4083985179086044,
+      "grad_norm": 34951.4765625,
+      "learning_rate": 3.681721262971413e-07,
+      "logits/chosen": -0.8185158371925354,
+      "logits/rejected": -0.8208706378936768,
+      "logps/chosen": -256.5106506347656,
+      "logps/ref_chosen": -238.3830108642578,
+      "logps/ref_rejected": -404.7466735839844,
+      "logps/rejected": -448.7279052734375,
+      "loss": 1711.1511,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.18127650022506714,
+      "rewards/margins": 0.2585355341434479,
+      "rewards/rejected": -0.4398120641708374,
+      "sft_loss": 0.9598148465156555,
+      "step": 31
+    },
+    {
+      "epoch": 0.4215726636475916,
+      "grad_norm": 22634.892578125,
+      "learning_rate": 3.577159362352426e-07,
+      "logits/chosen": -0.8045048713684082,
+      "logits/rejected": -0.871455192565918,
+      "logps/chosen": -298.8917236328125,
+      "logps/ref_chosen": -275.08062744140625,
+      "logps/ref_rejected": -439.5594787597656,
+      "logps/rejected": -486.42987060546875,
+      "loss": 1073.0205,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.23811063170433044,
+      "rewards/margins": 0.2305929809808731,
+      "rewards/rejected": -0.46870362758636475,
+      "sft_loss": 1.066259741783142,
+      "step": 32
+    },
+    {
+      "epoch": 0.43474680938657884,
+      "grad_norm": 17340.03515625,
+      "learning_rate": 3.470229631680624e-07,
+      "logits/chosen": -0.8264797329902649,
+      "logits/rejected": -0.8613673448562622,
+      "logps/chosen": -272.50433349609375,
+      "logps/ref_chosen": -253.18777465820312,
+      "logps/ref_rejected": -413.8665771484375,
+      "logps/rejected": -456.3052978515625,
+      "loss": 1156.1897,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.19316571950912476,
+      "rewards/margins": 0.23122143745422363,
+      "rewards/rejected": -0.4243871867656708,
+      "sft_loss": 0.9994233846664429,
+      "step": 33
+    },
+    {
+      "epoch": 0.44792095512556607,
+      "grad_norm": 18645.935546875,
+      "learning_rate": 3.361167125710832e-07,
+      "logits/chosen": -0.7793031334877014,
+      "logits/rejected": -0.8169309496879578,
+      "logps/chosen": -277.7130126953125,
+      "logps/ref_chosen": -257.9416809082031,
+      "logps/ref_rejected": -420.4779357910156,
+      "logps/rejected": -463.8733215332031,
+      "loss": 1071.5037,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.197713240981102,
+      "rewards/margins": 0.23624077439308167,
+      "rewards/rejected": -0.43395400047302246,
+      "sft_loss": 1.0031678676605225,
+      "step": 34
+    },
+    {
+      "epoch": 0.4610951008645533,
+      "grad_norm": 19952.755859375,
+      "learning_rate": 3.2502115875008516e-07,
+      "logits/chosen": -0.8148825764656067,
+      "logits/rejected": -0.8491742610931396,
+      "logps/chosen": -299.02423095703125,
+      "logps/ref_chosen": -274.88323974609375,
+      "logps/ref_rejected": -371.6654968261719,
+      "logps/rejected": -408.4618835449219,
+      "loss": 1289.3344,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.24141037464141846,
+      "rewards/margins": 0.12655334174633026,
+      "rewards/rejected": -0.3679637014865875,
+      "sft_loss": 1.0052714347839355,
+      "step": 35
+    },
+    {
+      "epoch": 0.47426924660354053,
+      "grad_norm": 27593.88671875,
+      "learning_rate": 3.137606921404191e-07,
+      "logits/chosen": -0.7783984541893005,
+      "logits/rejected": -0.8010607957839966,
+      "logps/chosen": -279.0986022949219,
+      "logps/ref_chosen": -259.29522705078125,
+      "logps/ref_rejected": -394.7854919433594,
+      "logps/rejected": -431.82940673828125,
+      "loss": 1168.3469,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.19803372025489807,
+      "rewards/margins": 0.17240510880947113,
+      "rewards/rejected": -0.3704387843608856,
+      "sft_loss": 0.929314136505127,
+      "step": 36
+    },
+    {
+      "epoch": 0.4874433923425278,
+      "grad_norm": 24689.4375,
+      "learning_rate": 3.0236006569153616e-07,
+      "logits/chosen": -0.7673581838607788,
+      "logits/rejected": -0.7947245836257935,
+      "logps/chosen": -277.85296630859375,
+      "logps/ref_chosen": -257.4971923828125,
+      "logps/ref_rejected": -428.3898620605469,
+      "logps/rejected": -473.2132873535156,
+      "loss": 1577.0131,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.20355747640132904,
+      "rewards/margins": 0.24467676877975464,
+      "rewards/rejected": -0.4482342600822449,
+      "sft_loss": 0.982255220413208,
+      "step": 37
+    },
+    {
+      "epoch": 0.500617538081515,
+      "grad_norm": 22088.283203125,
+      "learning_rate": 2.9084434045463254e-07,
+      "logits/chosen": -0.7526673674583435,
+      "logits/rejected": -0.8011683821678162,
+      "logps/chosen": -264.4170837402344,
+      "logps/ref_chosen": -247.0706024169922,
+      "logps/ref_rejected": -412.8132019042969,
+      "logps/rejected": -449.17218017578125,
+      "loss": 959.661,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.173464834690094,
+      "rewards/margins": 0.19012488424777985,
+      "rewards/rejected": -0.36358973383903503,
+      "sft_loss": 0.8840616345405579,
+      "step": 38
+    },
+    {
+      "epoch": 0.5137916838205022,
+      "grad_norm": 23851.294921875,
+      "learning_rate": 2.7923883049302066e-07,
+      "logits/chosen": -0.7483999729156494,
+      "logits/rejected": -0.7759736776351929,
+      "logps/chosen": -267.4944763183594,
+      "logps/ref_chosen": -248.76388549804688,
+      "logps/ref_rejected": -392.1750183105469,
+      "logps/rejected": -427.758056640625,
+      "loss": 1144.1176,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.1873060017824173,
+      "rewards/margins": 0.16852441430091858,
+      "rewards/rejected": -0.35583043098449707,
+      "sft_loss": 1.0190958976745605,
+      "step": 39
+    },
+    {
+      "epoch": 0.5269658295594895,
+      "grad_norm": 52302.57421875,
+      "learning_rate": 2.6756904723632324e-07,
+      "logits/chosen": -0.8017964959144592,
+      "logits/rejected": -0.8624761700630188,
+      "logps/chosen": -280.15875244140625,
+      "logps/ref_chosen": -259.4993896484375,
+      "logps/ref_rejected": -399.8374938964844,
+      "logps/rejected": -438.02447509765625,
+      "loss": 1198.8533,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.20659363269805908,
+      "rewards/margins": 0.17527592182159424,
+      "rewards/rejected": -0.3818695843219757,
+      "sft_loss": 0.9950222373008728,
+      "step": 40
+    },
+    {
+      "epoch": 0.5401399752984768,
+      "grad_norm": 32214.91796875,
+      "learning_rate": 2.5586064340081516e-07,
+      "logits/chosen": -0.775903046131134,
+      "logits/rejected": -0.8436408042907715,
+      "logps/chosen": -277.5977783203125,
+      "logps/ref_chosen": -257.62872314453125,
+      "logps/ref_rejected": -416.6236267089844,
+      "logps/rejected": -455.8535461425781,
+      "loss": 932.2411,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.19969075918197632,
+      "rewards/margins": 0.19260822236537933,
+      "rewards/rejected": -0.39229899644851685,
+      "sft_loss": 0.9193465709686279,
+      "step": 41
+    },
+    {
+      "epoch": 0.553314121037464,
+      "grad_norm": 18867.783203125,
+      "learning_rate": 2.4413935659918487e-07,
+      "logits/chosen": -0.7325506210327148,
+      "logits/rejected": -0.7629200220108032,
+      "logps/chosen": -278.38702392578125,
+      "logps/ref_chosen": -258.0796813964844,
+      "logps/ref_rejected": -403.37554931640625,
+      "logps/rejected": -441.0909729003906,
+      "loss": 1111.9651,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.20307336747646332,
+      "rewards/margins": 0.17408138513565063,
+      "rewards/rejected": -0.37715476751327515,
+      "sft_loss": 0.9346740245819092,
+      "step": 42
+    },
+    {
+      "epoch": 0.5664882667764513,
+      "grad_norm": 34268.01953125,
+      "learning_rate": 2.3243095276367684e-07,
+      "logits/chosen": -0.8410815000534058,
+      "logits/rejected": -0.8493119478225708,
+      "logps/chosen": -283.8769226074219,
+      "logps/ref_chosen": -263.16790771484375,
+      "logps/ref_rejected": -372.30419921875,
+      "logps/rejected": -405.7116394042969,
+      "loss": 1075.1367,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.20709021389484406,
+      "rewards/margins": 0.1269841194152832,
+      "rewards/rejected": -0.3340742886066437,
+      "sft_loss": 1.0096700191497803,
+      "step": 43
+    },
+    {
+      "epoch": 0.5796624125154385,
+      "grad_norm": 23751.416015625,
+      "learning_rate": 2.2076116950697937e-07,
+      "logits/chosen": -0.7676877379417419,
+      "logits/rejected": -0.7860767841339111,
+      "logps/chosen": -282.58770751953125,
+      "logps/ref_chosen": -260.87664794921875,
+      "logps/ref_rejected": -419.5744934082031,
+      "logps/rejected": -458.99755859375,
+      "loss": 1179.9371,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.2171107530593872,
+      "rewards/margins": 0.17712023854255676,
+      "rewards/rejected": -0.39423099160194397,
+      "sft_loss": 0.9293317794799805,
+      "step": 44
+    },
+    {
+      "epoch": 0.5928365582544257,
+      "grad_norm": 20057.634765625,
+      "learning_rate": 2.091556595453674e-07,
+      "logits/chosen": -0.810949981212616,
+      "logits/rejected": -0.807197630405426,
+      "logps/chosen": -303.0293884277344,
+      "logps/ref_chosen": -280.1766052246094,
+      "logps/ref_rejected": -414.77728271484375,
+      "logps/rejected": -454.41998291015625,
+      "loss": 1392.2808,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.22852776944637299,
+      "rewards/margins": 0.16789919137954712,
+      "rewards/rejected": -0.3964269459247589,
+      "sft_loss": 1.008779764175415,
+      "step": 45
+    },
+    {
+      "epoch": 0.606010703993413,
+      "grad_norm": 93258.9765625,
+      "learning_rate": 1.9763993430846392e-07,
+      "logits/chosen": -0.7731283903121948,
+      "logits/rejected": -0.8497406244277954,
+      "logps/chosen": -274.9454345703125,
+      "logps/ref_chosen": -256.38720703125,
+      "logps/ref_rejected": -440.72515869140625,
+      "logps/rejected": -478.4444580078125,
+      "loss": 1066.626,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.18558244407176971,
+      "rewards/margins": 0.19161023199558258,
+      "rewards/rejected": -0.3771926760673523,
+      "sft_loss": 1.0101566314697266,
+      "step": 46
+    },
+    {
+      "epoch": 0.6191848497324002,
+      "grad_norm": 25284.322265625,
+      "learning_rate": 1.862393078595809e-07,
+      "logits/chosen": -0.8175167441368103,
+      "logits/rejected": -0.8652117252349854,
+      "logps/chosen": -296.01422119140625,
+      "logps/ref_chosen": -273.3941650390625,
+      "logps/ref_rejected": -390.37255859375,
+      "logps/rejected": -426.4823913574219,
+      "loss": 1059.3556,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.22620078921318054,
+      "rewards/margins": 0.13489758968353271,
+      "rewards/rejected": -0.36109837889671326,
+      "sft_loss": 0.9877309203147888,
+      "step": 47
+    },
+    {
+      "epoch": 0.6323589954713874,
+      "grad_norm": 22755.78515625,
+      "learning_rate": 1.7497884124991485e-07,
+      "logits/chosen": -0.8109832406044006,
+      "logits/rejected": -0.8523979783058167,
+      "logps/chosen": -274.8309020996094,
+      "logps/ref_chosen": -256.0462951660156,
+      "logps/ref_rejected": -433.33880615234375,
+      "logps/rejected": -471.33514404296875,
+      "loss": 1024.0135,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": -0.18784590065479279,
+      "rewards/margins": 0.19211755692958832,
+      "rewards/rejected": -0.3799634277820587,
+      "sft_loss": 0.9346714019775391,
+      "step": 48
+    },
+    {
+      "epoch": 0.6455331412103746,
+      "grad_norm": 48471.87890625,
+      "learning_rate": 1.6388328742891678e-07,
+      "logits/chosen": -0.7700808048248291,
+      "logits/rejected": -0.8283007144927979,
+      "logps/chosen": -276.7380065917969,
+      "logps/ref_chosen": -258.337158203125,
+      "logps/ref_rejected": -402.8912658691406,
+      "logps/rejected": -438.73443603515625,
+      "loss": 1057.601,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1840084046125412,
+      "rewards/margins": 0.17442339658737183,
+      "rewards/rejected": -0.3584318161010742,
+      "sft_loss": 1.0198752880096436,
+      "step": 49
+    },
+    {
+      "epoch": 0.6587072869493619,
+      "grad_norm": 20364.625,
+      "learning_rate": 1.5297703683193753e-07,
+      "logits/chosen": -0.8074498176574707,
+      "logits/rejected": -0.8775460720062256,
+      "logps/chosen": -264.13189697265625,
+      "logps/ref_chosen": -246.30377197265625,
+      "logps/ref_rejected": -357.4236145019531,
+      "logps/rejected": -387.9647216796875,
+      "loss": 918.1786,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.17828112840652466,
+      "rewards/margins": 0.12712979316711426,
+      "rewards/rejected": -0.30541089177131653,
+      "sft_loss": 0.9230359196662903,
+      "step": 50
+    },
+    {
+      "epoch": 0.6718814326883491,
+      "grad_norm": 29058.451171875,
+      "learning_rate": 1.422840637647574e-07,
+      "logits/chosen": -0.7510178685188293,
+      "logits/rejected": -0.771770715713501,
+      "logps/chosen": -258.41845703125,
+      "logps/ref_chosen": -242.3370361328125,
+      "logps/ref_rejected": -373.0410461425781,
+      "logps/rejected": -403.00408935546875,
+      "loss": 951.6824,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.16081435978412628,
+      "rewards/margins": 0.13881637156009674,
+      "rewards/rejected": -0.299630731344223,
+      "sft_loss": 0.9331162571907043,
+      "step": 51
+    },
+    {
+      "epoch": 0.6850555784273363,
+      "grad_norm": 23650.7265625,
+      "learning_rate": 1.3182787370285865e-07,
+      "logits/chosen": -0.7598443031311035,
+      "logits/rejected": -0.7725738286972046,
+      "logps/chosen": -281.2547607421875,
+      "logps/ref_chosen": -261.9949645996094,
+      "logps/ref_rejected": -411.95611572265625,
+      "logps/rejected": -449.1616516113281,
+      "loss": 980.0952,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.19259795546531677,
+      "rewards/margins": 0.179457426071167,
+      "rewards/rejected": -0.3720553517341614,
+      "sft_loss": 0.9306559562683105,
+      "step": 52
+    },
+    {
+      "epoch": 0.6982297241663236,
+      "grad_norm": 49730.3984375,
+      "learning_rate": 1.2163145162128946e-07,
+      "logits/chosen": -0.8213989734649658,
+      "logits/rejected": -0.8896348476409912,
+      "logps/chosen": -287.832763671875,
+      "logps/ref_chosen": -268.6963806152344,
+      "logps/ref_rejected": -425.7789001464844,
+      "logps/rejected": -463.97308349609375,
+      "loss": 1025.2494,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.1913636326789856,
+      "rewards/margins": 0.19057826697826385,
+      "rewards/rejected": -0.38194188475608826,
+      "sft_loss": 0.9831621646881104,
+      "step": 53
+    },
+    {
+      "epoch": 0.7114038699053108,
+      "grad_norm": 39849.76171875,
+      "learning_rate": 1.1171721146870014e-07,
+      "logits/chosen": -0.7456237077713013,
+      "logits/rejected": -0.8003172278404236,
+      "logps/chosen": -271.5772399902344,
+      "logps/ref_chosen": -251.62240600585938,
+      "logps/ref_rejected": -423.13226318359375,
+      "logps/rejected": -464.643310546875,
+      "loss": 1263.4849,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.19954846799373627,
+      "rewards/margins": 0.21556226909160614,
+      "rewards/rejected": -0.4151107370853424,
+      "sft_loss": 0.9654486179351807,
+      "step": 54
+    },
+    {
+      "epoch": 0.724578015644298,
+      "grad_norm": 16490.654296875,
+      "learning_rate": 1.0210694689661939e-07,
+      "logits/chosen": -0.8203299045562744,
+      "logits/rejected": -0.8787042498588562,
+      "logps/chosen": -270.5526428222656,
+      "logps/ref_chosen": -251.76651000976562,
+      "logps/ref_rejected": -390.6857604980469,
+      "logps/rejected": -425.12744140625,
+      "loss": 909.9653,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.18786153197288513,
+      "rewards/margins": 0.1565551608800888,
+      "rewards/rejected": -0.34441667795181274,
+      "sft_loss": 0.9638007879257202,
+      "step": 55
+    },
+    {
+      "epoch": 0.7377521613832853,
+      "grad_norm": 21740.568359375,
+      "learning_rate": 9.282178335227883e-08,
+      "logits/chosen": -0.7758232951164246,
+      "logits/rejected": -0.8312428593635559,
+      "logps/chosen": -293.42498779296875,
+      "logps/ref_chosen": -272.2457580566406,
+      "logps/ref_rejected": -409.26513671875,
+      "logps/rejected": -448.3246154785156,
+      "loss": 1011.9142,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.21179243922233582,
+      "rewards/margins": 0.1788019835948944,
+      "rewards/rejected": -0.3905944228172302,
+      "sft_loss": 0.9946072697639465,
+      "step": 56
+    },
+    {
+      "epoch": 0.7509263071222725,
+      "grad_norm": 17951.88671875,
+      "learning_rate": 8.388213164029459e-08,
+      "logits/chosen": -0.7660011053085327,
+      "logits/rejected": -0.8385521173477173,
+      "logps/chosen": -278.1667175292969,
+      "logps/ref_chosen": -258.1634521484375,
+      "logps/ref_rejected": -446.6335754394531,
+      "logps/rejected": -487.20623779296875,
+      "loss": 921.0497,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.20003226399421692,
+      "rewards/margins": 0.2056940495967865,
+      "rewards/rejected": -0.4057263135910034,
+      "sft_loss": 0.9664478898048401,
+      "step": 57
+    },
+    {
+      "epoch": 0.7641004528612598,
+      "grad_norm": 34123.95703125,
+      "learning_rate": 7.530764305528958e-08,
+      "logits/chosen": -0.768264651298523,
+      "logits/rejected": -0.8065668344497681,
+      "logps/chosen": -280.6827392578125,
+      "logps/ref_chosen": -261.5390625,
+      "logps/ref_rejected": -408.1822509765625,
+      "logps/rejected": -444.2044677734375,
+      "loss": 973.7516,
+      "rewards/accuracies": 0.7890625,
+      "rewards/chosen": -0.19143663346767426,
+      "rewards/margins": 0.16878540813922882,
+      "rewards/rejected": -0.3602220416069031,
+      "sft_loss": 1.0135674476623535,
+      "step": 58
+    },
+    {
+      "epoch": 0.7772745986002471,
+      "grad_norm": 25089.880859375,
+      "learning_rate": 6.711716618408281e-08,
+      "logits/chosen": -0.7621768712997437,
+      "logits/rejected": -0.8304345607757568,
+      "logps/chosen": -285.82183837890625,
+      "logps/ref_chosen": -266.89788818359375,
+      "logps/ref_rejected": -408.42626953125,
+      "logps/rejected": -443.7984313964844,
+      "loss": 913.3936,
+      "rewards/accuracies": 0.7890625,
+      "rewards/chosen": -0.18923944234848022,
+      "rewards/margins": 0.16448232531547546,
+      "rewards/rejected": -0.3537217378616333,
+      "sft_loss": 1.0188053846359253,
+      "step": 59
+    },
+    {
+      "epoch": 0.7904487443392343,
+      "grad_norm": 24835.27734375,
+      "learning_rate": 5.932870547240454e-08,
+      "logits/chosen": -0.7458328008651733,
+      "logits/rejected": -0.8268925547599792,
+      "logps/chosen": -265.254638671875,
+      "logps/ref_chosen": -247.4517822265625,
+      "logps/ref_rejected": -384.67694091796875,
+      "logps/rejected": -417.5419006347656,
+      "loss": 1069.0216,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.1780286729335785,
+      "rewards/margins": 0.15062083303928375,
+      "rewards/rejected": -0.32864949107170105,
+      "sft_loss": 0.9329981803894043,
+      "step": 60
+    },
+    {
+      "epoch": 0.8036228900782215,
+      "grad_norm": 39707.6328125,
+      "learning_rate": 5.1959381647217665e-08,
+      "logits/chosen": -0.7766802906990051,
+      "logits/rejected": -0.8296952247619629,
+      "logps/chosen": -284.8126525878906,
+      "logps/ref_chosen": -264.2602233886719,
+      "logps/ref_rejected": -395.50848388671875,
+      "logps/rejected": -432.6766357421875,
+      "loss": 982.9723,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": -0.20552387833595276,
+      "rewards/margins": 0.1661575585603714,
+      "rewards/rejected": -0.37168145179748535,
+      "sft_loss": 0.9369614720344543,
+      "step": 61
+    },
+    {
+      "epoch": 0.8167970358172087,
+      "grad_norm": 19872.701171875,
+      "learning_rate": 4.502539408164385e-08,
+      "logits/chosen": -0.8309569358825684,
+      "logits/rejected": -0.8575015068054199,
+      "logps/chosen": -276.09942626953125,
+      "logps/ref_chosen": -255.97093200683594,
+      "logps/ref_rejected": -447.40618896484375,
+      "logps/rejected": -489.0849609375,
+      "loss": 1000.5587,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.20128466188907623,
+      "rewards/margins": 0.2155027985572815,
+      "rewards/rejected": -0.4167875051498413,
+      "sft_loss": 1.0023167133331299,
+      "step": 62
+    },
+    {
+      "epoch": 0.829971181556196,
+      "grad_norm": 16652.455078125,
+      "learning_rate": 3.854198518522564e-08,
+      "logits/chosen": -0.7885565757751465,
+      "logits/rejected": -0.8515602350234985,
+      "logps/chosen": -285.7626953125,
+      "logps/ref_chosen": -264.84906005859375,
+      "logps/ref_rejected": -413.2689514160156,
+      "logps/rejected": -451.171875,
+      "loss": 1036.8346,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.20913614332675934,
+      "rewards/margins": 0.16989250481128693,
+      "rewards/rejected": -0.3790286183357239,
+      "sft_loss": 0.9977405071258545,
+      "step": 63
+    },
+    {
+      "epoch": 0.8431453272951832,
+      "grad_norm": 95463.9609375,
+      "learning_rate": 3.2523406897802444e-08,
+      "logits/chosen": -0.7598262429237366,
+      "logits/rejected": -0.8194103240966797,
+      "logps/chosen": -297.930908203125,
+      "logps/ref_chosen": -274.2559814453125,
+      "logps/ref_rejected": -412.460693359375,
+      "logps/rejected": -450.8624267578125,
+      "loss": 1036.7638,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.23674927651882172,
+      "rewards/margins": 0.14726848900318146,
+      "rewards/rejected": -0.3840177655220032,
+      "sft_loss": 0.9848403930664062,
+      "step": 64
+    },
+    {
+      "epoch": 0.8563194730341704,
+      "grad_norm": 26849.763671875,
+      "learning_rate": 2.6982889360653376e-08,
+      "logits/chosen": -0.7949633002281189,
+      "logits/rejected": -0.854800820350647,
+      "logps/chosen": -282.3885803222656,
+      "logps/ref_chosen": -259.443115234375,
+      "logps/ref_rejected": -437.60064697265625,
+      "logps/rejected": -481.81732177734375,
+      "loss": 1209.8678,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.22945457696914673,
+      "rewards/margins": 0.21271246671676636,
+      "rewards/rejected": -0.4421670436859131,
+      "sft_loss": 0.9634147882461548,
+      "step": 65
+    },
+    {
+      "epoch": 0.8694936187731577,
+      "grad_norm": 24465.966796875,
+      "learning_rate": 2.1932611833775843e-08,
+      "logits/chosen": -0.8066317439079285,
+      "logits/rejected": -0.8882776498794556,
+      "logps/chosen": -263.16009521484375,
+      "logps/ref_chosen": -244.51507568359375,
+      "logps/ref_rejected": -397.60516357421875,
+      "logps/rejected": -433.0977478027344,
+      "loss": 996.5597,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.18644998967647552,
+      "rewards/margins": 0.168475940823555,
+      "rewards/rejected": -0.3549259305000305,
+      "sft_loss": 0.9470033049583435,
+      "step": 66
+    },
+    {
+      "epoch": 0.8826677645121449,
+      "grad_norm": 23544.0078125,
+      "learning_rate": 1.738367592322837e-08,
+      "logits/chosen": -0.8149206042289734,
+      "logits/rejected": -0.8741443753242493,
+      "logps/chosen": -278.1585388183594,
+      "logps/ref_chosen": -258.00384521484375,
+      "logps/ref_rejected": -426.8297119140625,
+      "logps/rejected": -466.4974365234375,
+      "loss": 974.3707,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.20154669880867004,
+      "rewards/margins": 0.1951308250427246,
+      "rewards/rejected": -0.39667749404907227,
+      "sft_loss": 0.9784504175186157,
+      "step": 67
+    },
+    {
+      "epoch": 0.8958419102511321,
+      "grad_norm": 37581.44140625,
+      "learning_rate": 1.3346081177391472e-08,
+      "logits/chosen": -0.8311805129051208,
+      "logits/rejected": -0.8835955858230591,
+      "logps/chosen": -267.4219665527344,
+      "logps/ref_chosen": -248.99069213867188,
+      "logps/ref_rejected": -397.907470703125,
+      "logps/rejected": -434.26910400390625,
+      "loss": 938.5573,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.1843128651380539,
+      "rewards/margins": 0.1793033629655838,
+      "rewards/rejected": -0.3636162281036377,
+      "sft_loss": 0.9809505343437195,
+      "step": 68
+    },
+    {
+      "epoch": 0.9090160559901194,
+      "grad_norm": 17509.748046875,
+      "learning_rate": 9.828703105789981e-09,
+      "logits/chosen": -0.8134777545928955,
+      "logits/rejected": -0.8525556325912476,
+      "logps/chosen": -267.26904296875,
+      "logps/ref_chosen": -247.39364624023438,
+      "logps/ref_rejected": -421.5614013671875,
+      "logps/rejected": -461.3177185058594,
+      "loss": 997.8082,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.19875425100326538,
+      "rewards/margins": 0.198808491230011,
+      "rewards/rejected": -0.39756274223327637,
+      "sft_loss": 0.9484153985977173,
+      "step": 69
+    },
+    {
+      "epoch": 0.9221902017291066,
+      "grad_norm": 29303.15625,
+      "learning_rate": 6.839273668796747e-09,
+      "logits/chosen": -0.7265700697898865,
+      "logits/rejected": -0.752987265586853,
+      "logps/chosen": -293.7417297363281,
+      "logps/ref_chosen": -269.7939758300781,
+      "logps/ref_rejected": -410.67901611328125,
+      "logps/rejected": -452.1363525390625,
+      "loss": 1046.5741,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.23947753012180328,
+      "rewards/margins": 0.1750955581665039,
+      "rewards/rejected": -0.414573073387146,
+      "sft_loss": 0.9382635354995728,
+      "step": 70
+    },
+    {
+      "epoch": 0.9353643474680938,
+      "grad_norm": 130671.40625,
+      "learning_rate": 4.384364281105973e-09,
+      "logits/chosen": -0.7530699968338013,
+      "logits/rejected": -0.823380708694458,
+      "logps/chosen": -284.096923828125,
+      "logps/ref_chosen": -263.344970703125,
+      "logps/ref_rejected": -426.18939208984375,
+      "logps/rejected": -464.8126525878906,
+      "loss": 1125.8024,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.20751963555812836,
+      "rewards/margins": 0.1787126660346985,
+      "rewards/rejected": -0.38623228669166565,
+      "sft_loss": 0.9687177538871765,
+      "step": 71
+    },
+    {
+      "epoch": 0.9485384932070811,
+      "grad_norm": 18457.01171875,
+      "learning_rate": 2.469371366337264e-09,
+      "logits/chosen": -0.7673330903053284,
+      "logits/rejected": -0.8481195569038391,
+      "logps/chosen": -281.8998107910156,
+      "logps/ref_chosen": -260.8071594238281,
+      "logps/ref_rejected": -384.2206115722656,
+      "logps/rejected": -420.32147216796875,
+      "loss": 963.4489,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.21092607080936432,
+      "rewards/margins": 0.15008221566677094,
+      "rewards/rejected": -0.36100828647613525,
+      "sft_loss": 0.9569988250732422,
+      "step": 72
+    },
+    {
+      "epoch": 0.9617126389460683,
+      "grad_norm": 21579.51171875,
+      "learning_rate": 1.0985044945254762e-09,
+      "logits/chosen": -0.7896211743354797,
+      "logits/rejected": -0.8543872833251953,
+      "logps/chosen": -287.5456848144531,
+      "logps/ref_chosen": -265.0205383300781,
+      "logps/ref_rejected": -415.27532958984375,
+      "logps/rejected": -454.45166015625,
+      "loss": 1135.3003,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.22525182366371155,
+      "rewards/margins": 0.16651120781898499,
+      "rewards/rejected": -0.39176300168037415,
+      "sft_loss": 1.0159409046173096,
+      "step": 73
+    },
+    {
+      "epoch": 0.9748867846850556,
+      "grad_norm": 34807.4765625,
+      "learning_rate": 2.7477712857215675e-10,
+      "logits/chosen": -0.7803131341934204,
+      "logits/rejected": -0.8543556928634644,
+      "logps/chosen": -276.9373779296875,
+      "logps/ref_chosen": -256.0310363769531,
+      "logps/ref_rejected": -395.774169921875,
+      "logps/rejected": -431.9486389160156,
+      "loss": 1060.5397,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.20906318724155426,
+      "rewards/margins": 0.15268170833587646,
+      "rewards/rejected": -0.36174488067626953,
+      "sft_loss": 1.023523211479187,
+      "step": 74
+    },
+    {
+      "epoch": 0.9880609304240429,
+      "grad_norm": 23669.484375,
+      "learning_rate": 0.0,
+      "logits/chosen": -0.7702288627624512,
+      "logits/rejected": -0.7675869464874268,
+      "logps/chosen": -276.1815185546875,
+      "logps/ref_chosen": -256.006103515625,
+      "logps/ref_rejected": -373.4369201660156,
+      "logps/rejected": -408.344482421875,
+      "loss": 925.0488,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": -0.20175378024578094,
+      "rewards/margins": 0.14732176065444946,
+      "rewards/rejected": -0.349075585603714,
+      "sft_loss": 0.9371762275695801,
+      "step": 75
+    },
+    {
+      "epoch": 0.9880609304240429,
+      "step": 75,
+      "total_flos": 0.0,
+      "train_loss": 1284.6718611653646,
+      "train_runtime": 5197.2653,
+      "train_samples_per_second": 1.869,
+      "train_steps_per_second": 0.014
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 75,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

Llama-3.2-3B-Instruct-ipo-proportion1.0-sft0.0-chunk7_6_rr_all/2/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3bc98d799dfa1223953bf5c33c0f52fb2dc0545c852e33c9ee57a04201c96c7
+size 7416